radelt: remove limit/offset in API calls

noi-techpark · Jul 3, 2024 · 4df22e5 · 4df22e5
1 parent 69a58b1
commit 4df22e5
Show file tree

Hide file tree

Showing 2 changed files with 36 additions and 53 deletions.
diff --git a/data-collectors/radelt/src/main/java/com/opendatahub/bdp/radelt/RadeltAPIClient.java b/data-collectors/radelt/src/main/java/com/opendatahub/bdp/radelt/RadeltAPIClient.java
@@ -25,12 +25,12 @@ public class RadeltAPIClient {
     private final String URL_CHALLENGES = "https://www.altoadigepedala.bz.it/dashboard/api/opendata/challenges";
     private final String URL_ORGANIZATIONS = "https://www.suedtirolradelt.bz.it/dashboard/api/opendata/organisations";
 
-    public AktionenResponseDto fetchChallenges(String active, String limit, String offset, String type)
+    public AktionenResponseDto fetchChallenges(String active, String type)
             throws Exception {
         URIBuilder uriBuilder = new URIBuilder(URL_CHALLENGES);
         uriBuilder.setParameter("active", active);
-        uriBuilder.setParameter("limit", limit);
-        uriBuilder.setParameter("offset", offset);
+        // uriBuilder.setParameter("limit", limit);
+        // uriBuilder.setParameter("offset", offset);
         uriBuilder.setParameter("type", type);
 
         URI uri = uriBuilder.build();
@@ -57,13 +57,13 @@ public AktionenResponseDto fetchChallenges(String active, String limit, String o
     }
 
     public OrganisationenResponseDto fetchOrganizations(String challengeId, String type,
-            String query, String limit, String offset) throws Exception {
+            String query) throws Exception {
         URIBuilder uriBuilder = new URIBuilder(URL_ORGANIZATIONS);
         uriBuilder.setParameter("challengeId", String.valueOf(challengeId));
         uriBuilder.setParameter("type", type);
         uriBuilder.setParameter("query", query);
-        uriBuilder.setParameter("limit", String.valueOf(limit));
-        uriBuilder.setParameter("offset", String.valueOf(offset));
+        // uriBuilder.setParameter("limit", String.valueOf(limit));
+        // uriBuilder.setParameter("offset", String.valueOf(offset));
 
         URI uri = uriBuilder.build();
         CloseableHttpClient httpClient = HttpClients.createDefault();

diff --git a/data-collectors/radelt/src/main/java/com/opendatahub/bdp/radelt/SyncScheduler.java b/data-collectors/radelt/src/main/java/com/opendatahub/bdp/radelt/SyncScheduler.java
@@ -85,56 +85,39 @@ public void syncJob() {
 		StationList stationsChallenges = new StationList();
 		DataMapDto<RecordDtoImpl> dataChallenges = new DataMapDto<>();
 
-		// Define base URL for challenges
-		int limitChallenges = 5;
-		int offsetChallenges = 0;
-
 		AktionenResponseDto challengeResponseDto;
-		do {
-			try {
-				challengeResponseDto = radelClient.fetchChallenges("true", String.valueOf(limitChallenges),
-						String.valueOf(offsetChallenges), "DISTANCE");
-
-				mappingUtilsAktionen.mapData(challengeResponseDto, this.actionCoordinates, stationsChallenges,
-						dataChallenges);
-				offsetChallenges += limitChallenges; // Increment offset for next page
-
-				for (RadeltChallengeDto challengeDto : challengeResponseDto.getData().getChallenges()) {
-					// Define base URL for organizations
-					// Initialize pagination variables
-					int limitOrganizations = 5; // Number of records per page
-					int offsetOrganizations = 0; // Starting offset
-
-					// Fetch and process organizations
-					OrganisationenResponseDto organizationResponseDto;
-					do {
-						try {
-							organizationResponseDto = radelClient.fetchOrganizations(
-									String.valueOf(challengeDto.getId()),
-									"", "", String.valueOf(limitOrganizations), String.valueOf(offsetOrganizations));
-
-							mappingUtilsOrganisationen.mapData(organizationResponseDto,
-									this.organizationCoordinates, stationsOrganisationen, dataOrganisationen);
-
-							// Update offset for next page
-							offsetOrganizations = offsetOrganizations + limitOrganizations;
-
-						} catch (Exception e) {
-							e.printStackTrace();
-							LOG.error("Error fetching organizations with challenge id {}. Error message: {}",
-									challengeDto.getId(),
-									e.getMessage());
-							return; // Exit the method if fetching organizations fails
-						}
-					} while (organizationResponseDto == null
-							|| organizationResponseDto.getData().getOrganisations().isEmpty());
+
+		try {
+			challengeResponseDto = radelClient.fetchChallenges("true", "DISTANCE");
+
+			mappingUtilsAktionen.mapData(challengeResponseDto, this.actionCoordinates, stationsChallenges,
+					dataChallenges);
+
+			for (RadeltChallengeDto challengeDto : challengeResponseDto.getData().getChallenges()) {
+				// Define base URL for organizations
+				// Initialize pagination variables
+				// Fetch and process organizations
+				OrganisationenResponseDto organizationResponseDto;
+				try {
+					organizationResponseDto = radelClient.fetchOrganizations(
+							String.valueOf(challengeDto.getId()),
+							"", "");
+
+					mappingUtilsOrganisationen.mapData(organizationResponseDto,
+							this.organizationCoordinates, stationsOrganisationen, dataOrganisationen);
+
+				} catch (Exception e) {
+					e.printStackTrace();
+					LOG.error("Error fetching organizations with challenge id {}. Error message: {}",
+							challengeDto.getId(),
+							e.getMessage());
+					return; // Exit the method if fetching organizations fails
 				}
-			} catch (Exception e) {
-				e.printStackTrace();
-				LOG.error("Error fetching challenges:", e.getMessage());
-				break; // Exit the loop if fetching challenges fails
 			}
-		} while (challengeResponseDto == null || challengeResponseDto.getData().getChallenges().isEmpty());
+		} catch (Exception e) {
+			e.printStackTrace();
+			LOG.error("Error fetching challenges:", e.getMessage());
+		}
 
 		// Sync with Open Data Hub
 		try {