Skip to content

Commit

Permalink
feat: add parsing date, add pagination on api call, change parameters…
Browse files Browse the repository at this point in the history 
… on api call
  • Loading branch information
@Giacomo92
Giacomo92 committed Jul 2, 2024
1 parent 4bcaa3e commit 567b9ce
Show file tree
Hide file tree
Showing 10 changed files with 223 additions and 77 deletions.
2 changes: 1 addition & 1 deletion data-collectors/radelt/.env.example
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ OAUTH_CLIENT_NAME=odh-mobility-datacollector-development
OAUTH_CLIENT_SECRET=7bd46f8f-c296-416d-a13d-dc81e68d0830

ODH_CLIENT_STATIONTYPE=your-stationtype
ODH_CLIENT_PROVENANCE_NAME=dc-helloworld
ODH_CLIENT_PROVENANCE_NAME=dc-radelt
ODH_CLIENT_PROVENANCE_VERSION=0.0.0-local-dev
ODH_CLIENT_PROVENANCE_ORIGIN=your-original-datasource
ODH_CLIENT_PERIOD=600
Expand Down
95 changes: 68 additions & 27 deletions data-collectors/radelt/src/main/java/com/opendatahub/bdp/radelt/SyncScheduler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import org.springframework.stereotype.Service;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.opendatahub.bdp.radelt.dto.aktionen.AktionenResponseDto;
import com.opendatahub.bdp.radelt.dto.aktionen.RadeltChallengeDto;
import com.opendatahub.bdp.radelt.dto.organisationen.OrganisationenResponseDto;
import com.opendatahub.bdp.radelt.dto.utils.MappingUtilsAktionen;
import com.opendatahub.bdp.radelt.dto.utils.MappingUtilsOrganisationen;
Expand Down Expand Up @@ -45,52 +46,82 @@ private void syncDataTypes() {
/**
* Scheduled job Aktionen
*/
@Scheduled(cron = "${scheduler.actions}")
@Scheduled(cron = "${scheduler.actions_and_organization}")
public void syncJobAktionen() {
LOG.info("Cron job syncJobAktionen started: Sync Stations with type {} and data types",
odhClient.getIntegreenTypology());
// Define base URL for challenges
String baseUrlChallenges = "https://www.altoadigepedala.bz.it/dashboard/api/opendata/challenges";
// Fetch and process challenges
int limit = 5;
int offset = 0;

AktionenResponseDto challengeResponseDto;
try {
challengeResponseDto = fetchChallenges(baseUrlChallenges, "true", "5", "0", "");
} catch (Exception e) {
LOG.error("Error fetching challenges: {}", e.getMessage());
return; // Exit the method if fetching challenges fails
}

MappingUtilsAktionen.mapToStationList(challengeResponseDto, odhClient, LOG);
while (true) {
try {
challengeResponseDto = fetchChallenges(baseUrlChallenges, "true", String.valueOf(limit), String.valueOf(offset), "DISTANCE");
if (challengeResponseDto == null) {
break; // No more data to fetch
}
MappingUtilsAktionen.mapToStationList(challengeResponseDto, odhClient, LOG);
offset += limit; // Increment offset for next page

for (RadeltChallengeDto challengeDto : challengeResponseDto.getData().getChallenges())
{
syncJobOrganisationen(challengeDto);
}
} catch (Exception e) {
e.printStackTrace();
LOG.error("Error fetching challenges:", e.getMessage());
break; // Exit the loop if fetching challenges fails
}
}

LOG.info("Cron job syncJobAktionen completed successfully");
}

/**
* Scheduled job Organisationen
*/
@Scheduled(cron = "${scheduler.organization}")
public void syncJobOrganisationen() {
LOG.info("Cron job syncJobOrganisationen started: Pushing measurements for {}",
odhClient.getIntegreenTypology());
private void syncJobOrganisationen(RadeltChallengeDto challengeDto) {

LOG.info("Cron job syncJobOrganisationen started: Pushing challenge with id: #" + challengeDto.getId(),
odhClient.getIntegreenTypology());

// Define base URL for organizations
String baseUrlOrganizations = "https://www.suedtirolradelt.bz.it/dashboard/api/opendata/organisations";
// Initialize pagination variables
String limit = "5"; // Number of records per page
String offset = "0"; // Starting offset

// Fetch and process organizations
OrganisationenResponseDto organizationResponseDto;
try {
organizationResponseDto = fetchOrganizations(baseUrlOrganizations, "280", "", "", "10", "0");
} catch (Exception e) {
LOG.error("Error fetching organizations: {}", e.getMessage());
return; // Exit the method if fetching organizations fails
}
while (true) {
try {
organizationResponseDto = fetchOrganizations(baseUrlOrganizations, String.valueOf(challengeDto.getId()), "", "", limit, offset);
if (organizationResponseDto == null) {
break; // Exit the loop if no more data
}

MappingUtilsOrganisationen.mapToStationList(organizationResponseDto, odhClient, LOG);
MappingUtilsOrganisationen.mapToStationList(organizationResponseDto, odhClient, LOG);

// Update offset for next page
offset = String.valueOf(Integer.parseInt(offset) + Integer.parseInt(limit));

} catch (Exception e) {
e.printStackTrace();
LOG.error("Error fetching organizations with challenge id #" + + challengeDto.getId() + " : ", e.getMessage());
return; // Exit the method if fetching organizations fails
}
}

LOG.info("Cron job Organisationen completed successfully");
}

public static AktionenResponseDto fetchChallenges(String baseUrl, String active, String limit, String offset,
String type) throws Exception {
public static AktionenResponseDto fetchChallenges(
String baseUrl,
String active,
String limit,
String offset,
String type
) throws Exception {
URIBuilder uriBuilder = new URIBuilder(baseUrl);
uriBuilder.setParameter("active", active);
uriBuilder.setParameter("limit", limit);
Expand All @@ -113,12 +144,21 @@ public static AktionenResponseDto fetchChallenges(String baseUrl, String active,
} else {
LOG.error("HTTP Request failed with status code: {}", response.getStatusLine().getStatusCode());
}
}catch (Exception e) {
e.printStackTrace();
throw new Exception("Error fetching actions", e);
}
return null;
}

public static OrganisationenResponseDto fetchOrganizations(String baseUrl, String challengeId, String type,
String query, String limit, String offset) throws Exception {
public static OrganisationenResponseDto fetchOrganizations(
String baseUrl,
String challengeId,
String type,
String query,
String limit,
String offset
) throws Exception {
URIBuilder uriBuilder = new URIBuilder(baseUrl);
uriBuilder.setParameter("challengeId", String.valueOf(challengeId));
uriBuilder.setParameter("type", type);
Expand All @@ -129,6 +169,7 @@ public static OrganisationenResponseDto fetchOrganizations(String baseUrl, Strin
URI uri = uriBuilder.build();
CloseableHttpClient httpClient = HttpClients.createDefault();
HttpGet request = new HttpGet(uri);
LOG.info(uri.toString());

try (CloseableHttpResponse response = httpClient.execute(request)) {
if (response.getStatusLine().getStatusCode() == 200) {
Expand Down
103 changes: 86 additions & 17 deletions ...tors/radelt/src/main/java/com/opendatahub/bdp/radelt/dto/aktionen/RadeltChallengeDto.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,18 +4,23 @@

package com.opendatahub.bdp.radelt.dto.aktionen;

import java.util.Date;
import com.fasterxml.jackson.annotation.JsonProperty;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.TimeZone;

public class RadeltChallengeDto {
private int id;
private String name;
private String shortName;
private String headerImage;
private Date start;
private Date end;
private Date registrationStart;
private Date registrationEnd;
private long start;
private long end;
private long registrationStart;
private long entryStart;
private long registrationEnd;
private long entryEnd;
private String type;
@JsonProperty("isExternal")
private boolean isExternal;
Expand Down Expand Up @@ -57,36 +62,100 @@ public void setHeaderImage(String headerImage) {
this.headerImage = headerImage;
}

public Date getStart() {
public long getStart() {
return start;
}

public void setStart(Date start) {
this.start = start;
public void setStart(String start) {
try {
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'");
sdf.setTimeZone(TimeZone.getTimeZone("UTC"));
// Parse the date string to a Date object
Date date = sdf.parse(start);
this.start = date.getTime();
} catch (ParseException e) {
e.printStackTrace();
}
}

public Date getEnd() {
public long getEntryStart() {
return entryStart;
}

public void setEntryStart(String entryStart) {
try{
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'");
sdf.setTimeZone(TimeZone.getTimeZone("UTC"));
// Parse the date string to a Date object
Date date = sdf.parse(entryStart);
this.entryStart = date.getTime();
} catch (ParseException e) {
e.printStackTrace();
}
}

public long getEnd() {
return end;
}

public void setEnd(Date end) {
this.end = end;
public void setEnd(String end) {
try {
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'");
sdf.setTimeZone(TimeZone.getTimeZone("UTC"));
// Parse the date string to a Date object
Date date = sdf.parse(end);
this.end = date.getTime();
} catch (ParseException e) {
e.printStackTrace();
}
}

public Date getRegistrationStart() {
public long getEntryEnd() {
return entryEnd;
}

public void setEntryEnd(String entryEnd) {
try {
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'");
sdf.setTimeZone(TimeZone.getTimeZone("UTC"));
// Parse the date string to a Date object
Date date = sdf.parse(entryEnd);
this.registrationStart = date.getTime();
} catch (ParseException e) {
e.printStackTrace();
}
}

public long getRegistrationStart() {
return registrationStart;
}

public void setRegistrationStart(Date registrationStart) {
this.registrationStart = registrationStart;
public void setRegistrationStart(String registrationStart) {
try {
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'");
sdf.setTimeZone(TimeZone.getTimeZone("UTC"));
// Parse the date string to a Date object
Date date = sdf.parse(registrationStart);
this.registrationStart = date.getTime();
} catch (ParseException e) {
e.printStackTrace();
}
}

public Date getRegistrationEnd() {
public long getRegistrationEnd() {
return registrationEnd;
}

public void setRegistrationEnd(Date registrationEnd) {
this.registrationEnd = registrationEnd;
public void setRegistrationEnd(String registrationEnd) {
try {
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'");
sdf.setTimeZone(TimeZone.getTimeZone("UTC"));
// Parse the date string to a Date object
Date date = sdf.parse(registrationEnd);
this.registrationEnd = date.getTime();
} catch (ParseException e) {
e.printStackTrace();
}
}

public String getType() {
Expand Down
22 changes: 15 additions & 7 deletions ...s/radelt/src/main/java/com/opendatahub/bdp/radelt/dto/aktionen/RadeltChallengeMetric.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,10 @@

package com.opendatahub.bdp.radelt.dto.aktionen;

import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.TimeZone;

public class RadeltChallengeMetric {
private long id;
Expand All @@ -20,7 +22,7 @@ public class RadeltChallengeMetric {
private double money_saved;
private Double physical_activity_percentage;
private int number_of_people;
private LocalDateTime created_at;
private long created_at;
private int organisation_count;
private int workplace_count;
private int school_count;
Expand Down Expand Up @@ -127,14 +129,20 @@ public void setNumber_of_people(int number_of_people) {
this.number_of_people = number_of_people;
}

public LocalDateTime getCreated_at() {
public long getCreated_at() {
return created_at;
}

public void setCreated_at(String created_at) {
DateTimeFormatter format = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSSSSX");
LocalDateTime formattedDate = LocalDateTime.parse(created_at, format);
this.created_at = formattedDate;
try {
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'");
sdf.setTimeZone(TimeZone.getTimeZone("UTC"));
// Parse the date string to a Date object
Date date = sdf.parse(created_at);
this.created_at = date.getTime();
} catch (ParseException e) {
e.printStackTrace();
}
}

public int getOrganisation_count() {
Expand Down
23 changes: 15 additions & 8 deletions .../main/java/com/opendatahub/bdp/radelt/dto/organisationen/RadeltChallengeStatisticDto.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,10 @@

package com.opendatahub.bdp.radelt.dto.organisationen;

import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.TimeZone;

public class RadeltChallengeStatisticDto {

Expand All @@ -21,7 +23,7 @@ public class RadeltChallengeStatisticDto {
private double money_saved;
private Object physical_activity_percentage; // can be null, so Object type
private int number_of_people;
private LocalDateTime created_at;
private long created_at;
private String name;
private String challenge_type;

Expand Down Expand Up @@ -121,15 +123,20 @@ public void setNumber_of_people(int number_of_people) {
this.number_of_people = number_of_people;
}

public LocalDateTime getCreated_at() {
public long getCreated_at() {
return created_at;
}

public void setCreated_at(String created_at) {

DateTimeFormatter format = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSSSSX");
LocalDateTime formattedDate = LocalDateTime.parse(created_at, format);
this.created_at = formattedDate;
try {
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'");
sdf.setTimeZone(TimeZone.getTimeZone("UTC"));
// Parse the date string to a Date object
Date date = sdf.parse(created_at);
this.created_at = date.getTime();
} catch (ParseException e) {
e.printStackTrace();
}
}

public String getName() {
Expand Down
Loading

0 comments on commit 567b9ce

Please sign in to comment.