import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.commons.lang3.StringUtils;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ArrayNode;
import edu.internet2.middleware.grouper.Group;
import edu.internet2.middleware.grouper.GroupFinder;
import edu.internet2.middleware.grouper.GroupSave;
import edu.internet2.middleware.grouper.GrouperSession;
import edu.internet2.middleware.grouper.Stem;
import edu.internet2.middleware.grouper.StemFinder;
import edu.internet2.middleware.grouper.app.loader.GrouperLoader;
import edu.internet2.middleware.grouper.app.loader.OtherJobScript;
import edu.internet2.middleware.grouper.cfg.GrouperConfig;
import edu.internet2.middleware.grouper.exception.GrouperSessionException;
import edu.internet2.middleware.grouper.group.TypeOfGroup;
import edu.internet2.middleware.grouper.misc.GrouperSessionHandler;
import edu.internet2.middleware.grouper.util.GrouperHttpClient;
import edu.internet2.middleware.grouper.util.GrouperHttpMethod;
import edu.internet2.middleware.grouper.util.GrouperUtil;
import edu.internet2.middleware.grouperClient.jdbc.GcDbAccess;
import edu.internet2.middleware.grouperClient.jdbc.tableSync.GcTableSyncFromData;
import edu.internet2.middleware.grouperClient.util.GrouperClientUtils;
//uncomment to compile in eclipse (and last line)
//public class Test58crashplan {
class TheState {
Map<String, Object> debugMap = new LinkedHashMap<String, Object>();
// String crashPlanClientId = "key-028abc-xyz-e09d8e";
// String crashPlanClientSecret = "abc";
// String crashPlanUrl = "https://console.us2.crashplan.com";
String crashPlanClientId = GrouperConfig.retrieveConfig().propertyValueString("crashPlanClientId");
String crashPlanClientSecret = GrouperConfig.retrieveConfig().propertyValueString("crashPlanClientSecret");
String crashPlanUrl = GrouperUtil.stripLastSlashIfExists(GrouperConfig.retrieveConfig().propertyValueString("crashPlanUrl"));
String crashPlanLocalEntityFolderName = "penn:isc:ait:apps:crashPlan:service:emailsNotMatched";
String crashPlanLoaderGroupNameOrg = "penn:isc:ait:apps:crashPlan:service:loader:orgLoader";
String crashPlanLoaderGroupNameRole = "penn:isc:ait:apps:crashPlan:service:loader:roleLoader";
String crashPlanLoaderGroupNameStatus = "penn:isc:ait:apps:crashPlan:service:loader:statusLoader";
String crashPlanLoaderGroupNameAdmin = "penn:isc:ait:apps:crashPlan:service:loader:adminLoader";
String crashPlanDeactiveUserGroupName = "penn:isc:ait:apps:crashPlan:service:policy:deactivateUsers";
String crashPlanBlockUserGroupName = "penn:isc:ait:apps:crashPlan:service:policy:blockUsers";
Stem crashPlanLocalEntityFolder = null;
int crashPlanTokenExpireMinutes = GrouperConfig.retrieveConfig().propertyValueInt("crashPlanTokenExpireMinutes", 10);
int crashPlanPageSize = GrouperConfig.retrieveConfig().propertyValueInt("crashPlanPageSize", 1000);
// admins, whether they are active or not
Set<BigDecimal> userIdsAdmins = new HashSet<>();
String currentToken = null;
long currentTokenRetrieved = -1;
int USER_ID_INDEX = 0;
int USER_UID_INDEX = 1;
int STATUS_ACTIVE_INDEX = 2;
int STATUS_BLOCKED_INDEX = 3;
int USERNAME_INDEX = 4;
int ORG_NAME_INDEX = 5;
int ADMIN_INDEX = 6;
int PENN_ID_INDEX = 7;
int EXTERNAL_SUBJECT_NAME_INDEX = 8;
List<Object[]> wsRows = new ArrayList<>();
List<String> userNames = new ArrayList<String>();
Map<BigDecimal, Set<String>> userIdToRoleNames = new HashMap<>();
GrouperSession grouperSession = null;
}
String retrieveAccessToken(TheState theState) {
if (theState.currentToken != null && (System.currentTimeMillis() - theState.currentTokenRetrieved) / (1000 * 60) < theState.crashPlanTokenExpireMinutes) {
GrouperUtil.mapAddValue(theState.debugMap, "accessTokenCache", 1);
return theState.currentToken;
}
GrouperUtil.mapAddValue(theState.debugMap, "accessTokenRetrieve", 1);
GrouperHttpClient grouperHttpClient = new GrouperHttpClient().addHeader("Accept", "application/json").
addHeader("Content-Type", "application/json").assignUser(theState.crashPlanClientId).assignPassword(theState.crashPlanClientSecret).
assignUrl(theState.crashPlanUrl + "/api/v3/oauth/token?grant_type=client_credentials").executeRequest();
//curl -X POST -k -H 'Accept: application/json' -H 'Content-Type: application/json' -H 'Authorization: Basic abc:xyz'
// -i 'https://console.us2.crashplan.com/api/v3/oauth/token?grant_type=client_credentials'
String responseBody = grouperHttpClient.getResponseBody();
if (grouperHttpClient.getResponseCode() != 200) {
throw new RuntimeException("Response code for retrieveToken: " + grouperHttpClient.getResponseCode() + ", " + responseBody);
}
JsonNode jsonJacksonNode = GrouperUtil.jsonJacksonNode(responseBody);
String accessToken = GrouperUtil.jsonJacksonGetString(jsonJacksonNode, "access_token");
GrouperUtil.assertion(!StringUtils.isBlank(accessToken), "Token is blank!");
theState.currentToken = accessToken;
theState.currentTokenRetrieved = System.currentTimeMillis();
return theState.currentToken;
}
void retrieveAdmins(TheState theState) {
int timeToLive=100000/theState.crashPlanPageSize;
int pgNum = 1;
while (true) {
// dont endless loop
if (timeToLive-- < 0) {
throw new RuntimeException("Endless loop");
}
// get an access token each time so it isnt expired
String accessToken = retrieveAccessToken(theState);
// make the call
GrouperHttpClient grouperHttpClient = new GrouperHttpClient().assignGrouperHttpMethod(GrouperHttpMethod.get).addHeader("Accept", "application/json").
addHeader("Content-Type", "application/json").addHeader("Authorization", "Bearer " + accessToken).
assignUrl(theState.crashPlanUrl + "/api/v1/User?admins=true&pgSize=" + theState.crashPlanPageSize + "&pgNum=" + pgNum).executeRequest();
// make sure valid response
String responseBody = grouperHttpClient.getResponseBody();
if (grouperHttpClient.getResponseCode() != 200) {
throw new RuntimeException("Response code for retrieve admins: " + grouperHttpClient.getResponseCode() + ", " + responseBody);
}
//{"metadata":{"timestamp":"2023-09-15T01:45:56.460Z","params":{"pgNum":"6","pgSize":"1000"}},"data":{"totalCount":4972,"users":[]}}
// make sure server supports page size
JsonNode jsonJacksonNode = GrouperUtil.jsonJacksonNode(responseBody);
int pageSizeFromResponse = GrouperUtil.jsonJacksonGetIntegerFromJsonPointer(jsonJacksonNode, "/metadata/params/pgSize");
GrouperUtil.assertion(pageSizeFromResponse == theState.crashPlanPageSize, "Page size different " + pageSizeFromResponse + " != " + theState.crashPlanPageSize);
// get the users
JsonNode usersNodeJsonNode = GrouperUtil.jsonJacksonGetNodeFromJsonPointer(jsonJacksonNode, "/data/users");
// if no users we are done
if (usersNodeJsonNode == null) {
break;
}
ArrayNode usersNode = (ArrayNode)usersNodeJsonNode;
if (usersNode.size() == 0) {
break;
}
for (int i=0;i<usersNode.size();i++) {
// {
// "userId":13224985,
JsonNode userNode = usersNode.get(i);
BigDecimal userId = new BigDecimal(GrouperUtil.jsonJacksonGetLong(userNode, "userId"));
theState.userIdsAdmins.add(userId);
}
pgNum++;
}
theState.debugMap.put("adminsRetrieve", GrouperUtil.length(theState.userIdsAdmins));
}
void retrieveUsers(TheState theState) {
int timeToLive=100000/theState.crashPlanPageSize;
int pgNum = 1;
while (true) {
// dont endless loop
if (timeToLive-- < 0) {
throw new RuntimeException("Endless loop");
}
// get an access token each time so it isnt expired
String accessToken = retrieveAccessToken(theState);
// make the call
GrouperHttpClient grouperHttpClient = new GrouperHttpClient().assignGrouperHttpMethod(GrouperHttpMethod.get).addHeader("Accept", "application/json").
addHeader("Content-Type", "application/json").addHeader("Authorization", "Bearer " + accessToken).
assignUrl(theState.crashPlanUrl + "/api/v1/User?incRoles=true&pgSize=" + theState.crashPlanPageSize + "&pgNum=" + pgNum).executeRequest();
// make sure valid response
String responseBody = grouperHttpClient.getResponseBody();
if (grouperHttpClient.getResponseCode() != 200) {
throw new RuntimeException("Response code: " + grouperHttpClient.getResponseCode() + ", " + responseBody);
}
//{"metadata":{"timestamp":"2023-09-15T01:45:56.460Z","params":{"pgNum":"6","pgSize":"1000"}},"data":{"totalCount":4972,"users":[]}}
// make sure server supports page size
JsonNode jsonJacksonNode = GrouperUtil.jsonJacksonNode(responseBody);
int pageSizeFromResponse = GrouperUtil.jsonJacksonGetIntegerFromJsonPointer(jsonJacksonNode, "/metadata/params/pgSize");
GrouperUtil.assertion(pageSizeFromResponse == theState.crashPlanPageSize, "Page size different " + pageSizeFromResponse + " != " + theState.crashPlanPageSize);
// get the users
JsonNode usersNodeJsonNode = GrouperUtil.jsonJacksonGetNodeFromJsonPointer(jsonJacksonNode, "/data/users");
// if no users we are done
if (usersNodeJsonNode == null) {
break;
}
ArrayNode usersNode = (ArrayNode)usersNodeJsonNode;
if (usersNode.size() == 0) {
break;
}
for (int i=0;i<usersNode.size();i++) {
JsonNode userNode = usersNode.get(i);
Object[] wsRow = new Object[9];
theState.wsRows.add(wsRow);
// int USER_ID_INDEX = 0;
// {
// "userId":13224985,
BigDecimal userId = new BigDecimal(GrouperUtil.jsonJacksonGetLong(userNode, "userId"));
wsRow[theState.USER_ID_INDEX] = userId;
// int USER_UID_INDEX = 1;
// "userUid":"933696115269786900",
wsRow[theState.USER_UID_INDEX] = GrouperUtil.jsonJacksonGetString(userNode, "userUid");
// int STATUS_ACTIVE_INDEX = 2;
// "active":true,
wsRow[theState.STATUS_ACTIVE_INDEX] = GrouperUtil.jsonJacksonGetBoolean(userNode, "active") ? "T" : "F";
// int STATUS_BLOCKED_INDEX = 3;
// "blocked":false,
wsRow[theState.STATUS_BLOCKED_INDEX] = GrouperUtil.jsonJacksonGetBoolean(userNode, "blocked") ? "T" : "F";
// int USERNAME_INDEX = 4;
// "username":"{60b78e88-ead8-445c-9cfd-0b87f74ea6cd}@wharton.upenn.edu",
wsRow[theState.USERNAME_INDEX] = GrouperUtil.jsonJacksonGetString(userNode, "username");
theState.userNames.add((String)wsRow[theState.USERNAME_INDEX]);
// "orgName":"CTS-Client Support",
// int ORG_NAME_INDEX = 5;
wsRow[theState.ORG_NAME_INDEX] = GrouperUtil.jsonJacksonGetString(userNode, "orgName");
// int ADMIN_INDEX = 6;
// "roles":[
// "org-admin",
// "desktop-user",
// "proe-user"
// ],
Set<String> roles = GrouperUtil.nonNull(GrouperUtil.jsonJacksonGetStringSet(userNode, "roles"));
boolean isAdmin = theState.userIdsAdmins.contains(wsRow[theState.USER_ID_INDEX]);
for (String role : GrouperUtil.nonNull(roles)) {
if (role.toLowerCase().contains("admin")) {
isAdmin=true;
}
}
wsRow[theState.ADMIN_INDEX] = isAdmin ? "T" : "F";
theState.userIdToRoleNames.put(userId, roles);
}
pgNum++;
}
}
public void retrievePennids(TheState theState) {
Map<String, String> pennkeyToPennid = new HashMap<String, String>();
List<Object[]> wsRowsFromPenn = new ArrayList<>();
for (Object[] wsRow : theState.wsRows) {
String username = (String)wsRow[theState.USERNAME_INDEX];
if (username.contains("@") && (username.endsWith("@upenn.edu") || username.endsWith(".upenn.edu") )) {
wsRowsFromPenn.add(wsRow);
}
}
int batchSize = 1000;
int numberOfBatches = GrouperUtil.batchNumberOfBatches(wsRowsFromPenn, batchSize, false);
for (int i=0;i<numberOfBatches;i++) {
List<Object[]> batchOfRows = GrouperUtil.batchList(wsRowsFromPenn, batchSize, i);
String sql = "select subject_id, subject_identifier0 from grouper_members where subject_source = 'pennperson' and subject_identifier0 in (" +
GrouperClientUtils.appendQuestions(GrouperUtil.length(batchOfRows)) + ")";
GcDbAccess gcDbAccess = new GcDbAccess().sql(sql);
for (Object[] wsRow : batchOfRows) {
String username = (String)wsRow[theState.USERNAME_INDEX];
String pennkey = GrouperUtil.prefixOrSuffix(username, "@", true);
gcDbAccess.addBindVar(pennkey);
}
List<Object[]> pennIdPennkeys = gcDbAccess.selectList(Object[].class);
for (Object[] pennIdPennkey : GrouperUtil.nonNull(pennIdPennkeys)) {
String pennId = (String)pennIdPennkey[0];
String pennkey = (String)pennIdPennkey[1];
pennkeyToPennid.put(pennkey, pennId);
}
}
theState.debugMap.put("pennIdsFound", pennkeyToPennid.size());
for (Object[] wsRow : wsRowsFromPenn) {
String username = (String)wsRow[theState.USERNAME_INDEX];
String pennkey = GrouperUtil.prefixOrSuffix(username, "@", true);
String pennId = pennkeyToPennid.get(pennkey);
if (!StringUtils.isBlank(pennId)) {
wsRow[theState.PENN_ID_INDEX] = pennId;
}
}
}
public void retrieveExistingLocalEntities(TheState theState) {
theState.crashPlanLocalEntityFolder = StemFinder.findByName(theState.grouperSession, theState.crashPlanLocalEntityFolderName, true);
Map<String, String> displayExtensionToName = new HashMap<>();
List<Object[]> nameDisplayExtensions = new GcDbAccess().
sql("select name, display_extension from grouper_groups gg where gg.type_of_group = 'entity' and gg.parent_stem = ?").
addBindVar(theState.crashPlanLocalEntityFolder.getId()).selectList(Object[].class);
// put existing local entities in a map
for (Object[] nameDisplayExtension : nameDisplayExtensions) {
String name = (String)nameDisplayExtension[0];
String displayExtension = (String)nameDisplayExtension[1];
displayExtensionToName.put(displayExtension, name);
}
theState.debugMap.put("existingLocalEntities", displayExtensionToName.size());
int matchedRows = 0;
// see which accounts match
for (Object[] wsRow : theState.wsRows) {
String pennId = (String)wsRow[theState.PENN_ID_INDEX];
if (!StringUtils.isBlank(pennId)) {
continue;
}
String email = (String)wsRow[theState.USERNAME_INDEX];
String name = displayExtensionToName.get(email);
if (!StringUtils.isBlank(name)) {
wsRow[theState.EXTERNAL_SUBJECT_NAME_INDEX] = name;
matchedRows++;
}
}
theState.debugMap.put("existingLocalEntityMatches", matchedRows);
// remove matches
for (Object[] wsRow : theState.wsRows) {
String email = (String)wsRow[theState.USERNAME_INDEX];
String externalSubjectName = (String)wsRow[theState.EXTERNAL_SUBJECT_NAME_INDEX];
if (!StringUtils.isBlank(externalSubjectName)) {
displayExtensionToName.remove(email);
}
}
//delete unused
theState.debugMap.put("deletedLocalEntities", displayExtensionToName.size());
int logSize = 100;
for (String externalSubjectName : displayExtensionToName.values()) {
Group localEntity = GroupFinder.findByName(externalSubjectName, true);
localEntity.delete();
if (OtherJobScript.retrieveFromThreadLocal() != null) {
OtherJobScript.retrieveFromThreadLocal().getOtherJobInput().getHib3GrouperLoaderLog().addDeleteCount(1);
}
if (logSize-- > 0) {
theState.debugMap.put("deleted_" + GrouperUtil.extensionFromName(externalSubjectName), true);
}
}
}
public void createMissingLocalEntities(TheState theState) {
int newLocalEntities = 0;
// see which accounts need a local entity
for (Object[] wsRow : theState.wsRows) {
String email = (String)wsRow[theState.USERNAME_INDEX];
String pennId = (String)wsRow[theState.PENN_ID_INDEX];
String externalSubjectName = (String)wsRow[theState.EXTERNAL_SUBJECT_NAME_INDEX];
if (StringUtils.isBlank(pennId) && StringUtils.isBlank(externalSubjectName)) {
String extension = email.replaceAll("[^a-zA-Z0-9_-]", "_");
Group group = new GroupSave().assignName(theState.crashPlanLocalEntityFolderName + ":" + extension).
assignDisplayExtension(email).assignTypeOfGroup(TypeOfGroup.entity).save();
if (OtherJobScript.retrieveFromThreadLocal() != null) {
OtherJobScript.retrieveFromThreadLocal().getOtherJobInput().getHib3GrouperLoaderLog().addInsertCount(1);
}
wsRow[theState.EXTERNAL_SUBJECT_NAME_INDEX] = group.getName();
newLocalEntities++;
}
}
theState.debugMap.put("localEntitesCreated", newLocalEntities);
}
public void syncWsRows(TheState theState) {
List<String> columnNames = GrouperUtil.toList("user_id", "user_uid", "status_active", "status_blocked",
"username", "org_name", "admin", "penn_id", "external_subject_name");
List<String> columnNamesPrimaryKey = GrouperUtil.toList("user_id");
new GcTableSyncFromData().assignDebugMap(theState.debugMap).assignDebugMapPrefix("user_").assignConnectionName("grouper").assignTableName("penn_crashplan_user")
.assignColumnNames(columnNames).assignColumnNamesPrimaryKey(columnNamesPrimaryKey).assignData(theState.wsRows).sync();
if (OtherJobScript.retrieveFromThreadLocal() != null) {
OtherJobScript.retrieveFromThreadLocal().getOtherJobInput().getHib3GrouperLoaderLog().addTotalCount(GrouperUtil.length(theState.wsRows));
}
columnNames = GrouperUtil.toList("user_id", "role_name");
columnNamesPrimaryKey = GrouperUtil.toList("user_id, role_name");
List<Object[]> userIdRoleNames = new ArrayList<>();
for (BigDecimal userId : theState.userIdToRoleNames.keySet()) {
Set<String> roles = theState.userIdToRoleNames.get(userId);
for (String role : roles) {
userIdRoleNames.add(GrouperUtil.toArrayObject(userId, role));
}
}
new GcTableSyncFromData().assignDebugMap(theState.debugMap).assignDebugMapPrefix("role_").assignConnectionName("grouper").assignTableName("penn_crashplan_role")
.assignColumnNames(columnNames).assignColumnNamesPrimaryKey(columnNamesPrimaryKey).assignData(userIdRoleNames).sync();
if (OtherJobScript.retrieveFromThreadLocal() != null) {
OtherJobScript.retrieveFromThreadLocal().getOtherJobInput().getHib3GrouperLoaderLog().addTotalCount(GrouperUtil.length(userIdRoleNames));
OtherJobScript.retrieveFromThreadLocal().getOtherJobInput().getHib3GrouperLoaderLog().
addInsertCount(GrouperUtil.intValue(theState.debugMap.get("insertsCount"), 0));
OtherJobScript.retrieveFromThreadLocal().getOtherJobInput().getHib3GrouperLoaderLog().
addUpdateCount(GrouperUtil.intValue(theState.debugMap.get("updatesCount"), 0));
OtherJobScript.retrieveFromThreadLocal().getOtherJobInput().getHib3GrouperLoaderLog().
addDeleteCount(GrouperUtil.intValue(theState.debugMap.get("deletesCount"), 0));
}
}
public void runLoaders(TheState theState) {
// these will run in this process since this is a daemon itself and is running on the daemon server
Group crashPlanLoaderGroupRole = GroupFinder.findByName(theState.crashPlanLoaderGroupNameRole, true);
GrouperLoader.runJobOnceForGroup(theState.grouperSession, crashPlanLoaderGroupRole, false);
Group crashPlanLoaderGroupAdmin = GroupFinder.findByName(theState.crashPlanLoaderGroupNameAdmin, true);
GrouperLoader.runJobOnceForGroup(theState.grouperSession, crashPlanLoaderGroupAdmin, false);
Group crashPlanLoaderGroupOrg = GroupFinder.findByName(theState.crashPlanLoaderGroupNameOrg, true);
GrouperLoader.runJobOnceForGroup(theState.grouperSession, crashPlanLoaderGroupOrg, false);
Group crashPlanLoaderGroupStatus = GroupFinder.findByName(theState.crashPlanLoaderGroupNameStatus, true);
GrouperLoader.runJobOnceForGroup(theState.grouperSession, crashPlanLoaderGroupStatus, false);
}
public void blockAndDeactivateUsers(TheState theState) {
Group usersToBlock = GroupFinder.findByName(theState.crashPlanBlockUserGroupName, true);
Group usersToDeactivate = GroupFinder.findByName(theState.crashPlanDeactiveUserGroupName, true);
//
// // dont endless loop
// if (timeToLive-- < 0) {
// throw new RuntimeException("Endless loop");
// }
//
// // get an access token each time so it isnt expired
// String accessToken = retrieveAccessToken(theState);
//
// // make the call
// GrouperHttpClient grouperHttpClient = new GrouperHttpClient().assignGrouperHttpMethod(GrouperHttpMethod.get).addHeader("Accept", "application/json").
// addHeader("Content-Type", "application/json").addHeader("Authorization", "Bearer " + accessToken).
// assignUrl(theState.crashPlanUrl + "/api/v1/User?incRoles=true&pgSize=" + theState.crashPlanPageSize + "&pgNum=" + pgNum).executeRequest();
//
// // make sure valid response
// String responseBody = grouperHttpClient.getResponseBody();
// if (grouperHttpClient.getResponseCode() != 200) {
// throw new RuntimeException("Response code: " + grouperHttpClient.getResponseCode() + ", " + responseBody);
// }
}
public void runLogic() {
final TheState theState = new TheState();
try {
GrouperSession.internal_callbackRootGrouperSession(new GrouperSessionHandler() {
@Override
public Object callback(GrouperSession theGrouperSession) throws GrouperSessionException {
theState.grouperSession = theGrouperSession;
retrieveAdmins(theState);
retrieveUsers(theState);
retrievePennids(theState);
retrieveExistingLocalEntities(theState);
createMissingLocalEntities(theState);
syncWsRows(theState);
runLoaders(theState);
//blockAndDeactivateUsers(theState);
return null;
}
});
} catch (Exception e) {
theState.debugMap.put("exception", GrouperUtil.getFullStackTrace(e));
throw new RuntimeException(e);
} finally {
if (OtherJobScript.retrieveFromThreadLocal() != null) {
OtherJobScript.retrieveFromThreadLocal().getOtherJobInput().getHib3GrouperLoaderLog().appendJobMessage(GrouperUtil.mapToString(theState.debugMap));
} else {
System.out.println(GrouperUtil.mapToString(theState.debugMap));
}
}
}
runLogic();
// public static void main(String[] args) {
// new Test58crashplan().runLogic();
// }
//
//}