dvc.yaml

stages:
  upload_zoning_docs:
    cmd: bash zoning/data_processing/upload_zoning_docs.sh
    deps:
    - zoning/data_processing/upload_zoning_docs.sh
    - data/orig-documents
    outs:
    - data/orig_documents_s3_manifest.json
  extract_text:
    cmd: python -m zoning.data_processing.extract_text
    deps:
    - zoning/data_processing/extract_text.py
    - data/orig_documents_s3_manifest.json
    params:
    - extract_text.orig_document_s3_bucket
    outs:
    - data/textract_dataset
  generate_dataset:
    cmd: python -m zoning.data_processing.generate_dataset
    deps:
    - zoning/data_processing/generate_dataset.py
    - data/names_all_towns.json
    - data/textract_dataset
    params:
    - generate_dataset.seed
    - generate_dataset.test_split_frac
    - publish_datasets
    outs:
    - data/hf_dataset
    - data/parquet_dataset
  generate_text_dataset:
    cmd: python -m zoning.data_processing.generate_text_dataset
    deps:
    - zoning/data_processing/generate_text_dataset.py
    - data/hf_dataset
    params:
    - publish_datasets
    outs:
    - data/hf_text_dataset
  index_towns:
    cmd: python -m zoning.data_processing.index_towns
    deps:
    - zoning/data_processing/index_towns.py
    - data/hf_text_dataset
  evaluate:
    cmd: python -m zoning.data_processing.eval ${eval}
    deps:
    - zoning/data_processing/eval.py
    - templates/extraction_chat_completion.pmpt.tpl
    - templates/extraction_completion.pmpt.tpl
    outs:
    - data/results/eval.parquet
    metrics:
    - data/results/eval.yaml:
        cache: true