Run De-dupe #125
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: Run De-dupe | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| geospatial: | |
| type: boolean | |
| description: Check if organization has geospatial duplicates | |
| required: true | |
| dryrun: | |
| type: boolean | |
| description: Only run the dryrun | |
| required: true | |
| default: true | |
| organization: | |
| description: 'Organization' | |
| required: true | |
| default: 'doi-gov' | |
| type: choice | |
| options: | |
| - allegheny-county-city-of-pittsburgh-western-pa-regional-data-center | |
| - americorps-gov | |
| - arkansas-gov | |
| - arlington-county | |
| - board-of-governors-of-the-federal-reserve-system | |
| - ca-gov | |
| - cdatribe-nsn-gov | |
| - cdip-ucsd-edu | |
| - census-gov | |
| - cfpb-gov | |
| - city-and-county-of-durham-north-carolina | |
| - city-of-austin | |
| - city-of-baltimore | |
| - city-of-baton-rouge | |
| - city-of-bloomington | |
| - city-of-boise | |
| - city-of-charlotte | |
| - city-of-chesapeake | |
| - city-of-chicago | |
| - city-of-ferndale-michigan | |
| - city-of-hartford | |
| - city-of-honolulu | |
| - city-of-jackson-mississippi | |
| - city-of-los-angeles | |
| - city-of-new-orleans | |
| - city-of-new-york | |
| - city-of-philadelphia | |
| - city-of-providence | |
| - city-of-san-francisco | |
| - city-of-santa-rosa | |
| - city-of-seattle | |
| - city-of-sioux-falls | |
| - city-of-somerville | |
| - city-of-tempe | |
| - coa-gatech-edu | |
| - commodity-futures-trading-commission | |
| - cook-county-of-illinois | |
| - cpsc-gov | |
| - dhs-gov | |
| - district-of-columbia | |
| - doc-gov | |
| - dod-gov | |
| - doe-gov | |
| - doi-gov | |
| - doj-gov | |
| - dol-gov | |
| - dot-gov | |
| - eac-gov | |
| - ed-gov | |
| - edac-unm-edu | |
| - eeoc-gov | |
| - eop-gov | |
| - epa-gov | |
| - exim-gov | |
| - fairfax-county-virginia | |
| - fcc-gov | |
| - fdic-gov | |
| - fec-gov | |
| - federal-laboratory-consortium | |
| - federal-retirement-thrift-investment-board | |
| - federal-trade-commission | |
| - fema-gov | |
| - fgdc-gov | |
| - fhfa-gov | |
| - florida-department-of-environmental-protection | |
| - gsa-gov | |
| - gulf-coast-ecosystem-restoration-council | |
| - hhs-gov | |
| - hud-gov | |
| - ibwc-gov | |
| - imls-gov | |
| - king-county-washington | |
| - lake-county-illinois | |
| - legal-services-corporation | |
| - library-of-congress | |
| - loudoun-county-virginia | |
| - louisville-metro-government | |
| - mcc-gov | |
| - montgomery-county-of-maryland | |
| - mspb-gov | |
| - municipality-of-anchorage | |
| - nara-gov | |
| - nasa-gov | |
| - national-credit-union-administration | |
| - national-institute-of-standards-and-technology | |
| - nc-gov | |
| - nd-gov | |
| - neh-gov | |
| - nga-mil | |
| - nitrd-gov | |
| - noaa-gov | |
| - nrc-gov | |
| - nsf-gov | |
| - ntsb-gov | |
| - office-of-management-and-budget | |
| - onhir-gov | |
| - opentopography | |
| - opm-gov | |
| - pbgc-gov | |
| - pennsylvania-geospatial-data-clearinghouse | |
| - rrb-gov | |
| - sandbox-organization | |
| - sba-gov | |
| - sec-gov | |
| - ssa-gov | |
| - state-gov | |
| - state-mn-us | |
| - state-of-alaska | |
| - state-of-connecticut | |
| - state-of-hawaii | |
| - state-of-illinois | |
| - state-of-iowa | |
| - state-of-maryland | |
| - state-of-missouri | |
| - state-of-new-york | |
| - state-of-oklahoma | |
| - state-of-oregon | |
| - state-of-washington | |
| - town-of-cary-north-carolina | |
| - town-of-chapel-hill-north-carolina | |
| - treasury-gov | |
| - u-s-department-of-labor-bureau-of-labor-statistics | |
| - u-washington-edu | |
| - uidaho-edu | |
| - usace-army-mil | |
| - usaid-gov | |
| - usda-gov | |
| - usgs-gov | |
| - usitc-gov | |
| - va-gov | |
| - vcgi-org | |
| - wake-county | |
| - wvu-edu | |
| jobs: | |
| dedupe: | |
| name: Run de-dupe | |
| runs-on: ubuntu-latest | |
| environment: prod | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v2 | |
| - name: Set up Python 3.8 | |
| uses: actions/setup-python@v1 | |
| with: { python-version: 3.8 } | |
| - name: Install pip and pipenv | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install pipenv | |
| - name: build | |
| run: pipenv sync | |
| - name: run dedupe with proper flags | |
| run: | | |
| args='' | |
| if ${{ inputs.geospatial }}; then | |
| args="${args} --geospatial" | |
| fi | |
| if ! ${{ inputs.dryrun }}; then | |
| args="${args} --commit" | |
| args="${args} --api-key ${{secrets.CKAN_API_TOKEN}}" | |
| fi | |
| pipenv run python duplicates-identifier-api.py $args ${{ inputs.organization }} |