From e78d0391e0a20f5d32d59bf94d6de7eeea548e30 Mon Sep 17 00:00:00 2001 From: Ethan White Date: Wed, 11 Jan 2017 21:16:17 -0500 Subject: [PATCH] Using `missingValues` instead of `nulls` in json files This is being added to the datapackage spec. See: https://github.com/frictionlessdata/specs/issues/353 --- lib/compile.py | 2 +- scripts/Adler2007.json | 6 +++--- scripts/Adult.json | 4 ++-- scripts/AmnioteDB.json | 2 +- scripts/AvianBodySize.json | 2 +- scripts/CForBioData.json | 2 +- scripts/FishParasiteHosts.json | 2 +- scripts/FrayJorge.json | 2 +- scripts/HomeRanges.json | 2 +- scripts/MammalDiet.json | 2 +- scripts/MammalMR2010.json | 2 +- scripts/MarineSize.json | 2 +- scripts/Matter2014.json | 8 ++++---- scripts/MoM2003.json | 2 +- scripts/Palmer2007.json | 2 +- scripts/PhytoplankonBiovolume.json | 12 ++++++------ scripts/PortalMammals.json | 2 +- scripts/Ramesh2010.json | 2 +- scripts/SonoranPerennials.json | 18 +++++++++--------- scripts/Steppe_plants_2013.json | 16 ++++++++-------- scripts/TreeWesternGhats.json | 4 ++-- scripts/Woods2009.json | 8 ++++---- test/test_retriever.py | 2 +- 23 files changed, 53 insertions(+), 53 deletions(-) diff --git a/lib/compile.py b/lib/compile.py index 79447c99f..85ab9ab03 100644 --- a/lib/compile.py +++ b/lib/compile.py @@ -29,7 +29,7 @@ def add_dialect(table_dict, table): for (key, val) in table['dialect'].items(): # dialect related key-value pairs # copied as is - if key == "nulls": + if key == "missingValues": table_dict[ 'cleanup'] = "Cleanup(correct_invalid_value, nulls=" + str(val) + ")" diff --git a/scripts/Adler2007.json b/scripts/Adler2007.json index b83f41be0..26bfd43e8 100644 --- a/scripts/Adler2007.json +++ b/scripts/Adler2007.json @@ -24,7 +24,7 @@ }, { "dialect": { - "nulls": [ + "missingValues": [ "NA" ] }, @@ -40,7 +40,7 @@ }, { "dialect": { - "nulls": [ + "missingValues": [ "NA" ], "replace_columns": [ @@ -96,7 +96,7 @@ }, { "dialect": { - "nulls": [ + "missingValues": [ "NA" ], "replace_columns": [ diff --git a/scripts/Adult.json b/scripts/Adult.json index ba450df82..90a7389bc 100644 --- a/scripts/Adult.json +++ b/scripts/Adult.json @@ -12,7 +12,7 @@ "dialect": { "delimiter": ",", "header_rows": "0", - "nulls": "?" + "missingValues": "?" }, "name": "data", "schema": { @@ -94,7 +94,7 @@ "dialect": { "delimiter": ",", "header_rows": "1", - "nulls": "?" + "missingValues": "?" }, "name": "test", "schema": { diff --git a/scripts/AmnioteDB.json b/scripts/AmnioteDB.json index e530121f3..06662d112 100644 --- a/scripts/AmnioteDB.json +++ b/scripts/AmnioteDB.json @@ -11,7 +11,7 @@ { "dialect": { "delimiter": ",", - "nulls": [ + "missingValues": [ -999 ] }, diff --git a/scripts/AvianBodySize.json b/scripts/AvianBodySize.json index 8b1b70fb5..0765b252f 100644 --- a/scripts/AvianBodySize.json +++ b/scripts/AvianBodySize.json @@ -10,7 +10,7 @@ "resources": [ { "dialect": { - "nulls": [ + "missingValues": [ -999 ] }, diff --git a/scripts/CForBioData.json b/scripts/CForBioData.json index dcfa15555..9dbba5cda 100644 --- a/scripts/CForBioData.json +++ b/scripts/CForBioData.json @@ -9,7 +9,7 @@ "resources": [ { "dialect": { - "nulls": [ + "missingValues": [ "NA" ] }, diff --git a/scripts/FishParasiteHosts.json b/scripts/FishParasiteHosts.json index 9460cde9d..3dd0440c2 100644 --- a/scripts/FishParasiteHosts.json +++ b/scripts/FishParasiteHosts.json @@ -5,7 +5,7 @@ "resources": [ { "dialect": { - "nulls": [ + "missingValues": [ "na" ], "replace_columns": [ diff --git a/scripts/FrayJorge.json b/scripts/FrayJorge.json index 7ee9fa770..e7ae50ac4 100644 --- a/scripts/FrayJorge.json +++ b/scripts/FrayJorge.json @@ -233,7 +233,7 @@ }, { "dialect": { - "nulls": [ + "missingValues": [ ".", "n/a" ] diff --git a/scripts/HomeRanges.json b/scripts/HomeRanges.json index 50f5fdb5d..8ea1c8b0f 100644 --- a/scripts/HomeRanges.json +++ b/scripts/HomeRanges.json @@ -9,7 +9,7 @@ "resources": [ { "dialect": { - "nulls": [ + "missingValues": [ "NA" ] }, diff --git a/scripts/MammalDiet.json b/scripts/MammalDiet.json index 3f1bcf69b..67dab58cb 100644 --- a/scripts/MammalDiet.json +++ b/scripts/MammalDiet.json @@ -10,7 +10,7 @@ { "dialect": { "contains_pk": "True", - "nulls": [ + "missingValues": [ "NA" ] }, diff --git a/scripts/MammalMR2010.json b/scripts/MammalMR2010.json index 9cdf5fea7..5e02bb999 100644 --- a/scripts/MammalMR2010.json +++ b/scripts/MammalMR2010.json @@ -10,7 +10,7 @@ "resources": [ { "dialect": { - "nulls": [ + "missingValues": [ -9999 ] }, diff --git a/scripts/MarineSize.json b/scripts/MarineSize.json index 35f6e6c15..d5b076cd8 100644 --- a/scripts/MarineSize.json +++ b/scripts/MarineSize.json @@ -10,7 +10,7 @@ "resources": [ { "dialect": { - "nulls": [ + "missingValues": [ "n/a", "0.0000E+00" ] diff --git a/scripts/Matter2014.json b/scripts/Matter2014.json index 89a38a390..6b26502aa 100644 --- a/scripts/Matter2014.json +++ b/scripts/Matter2014.json @@ -10,7 +10,7 @@ "resources": [ { "dialect": { - "nulls": [ + "missingValues": [ "n/a" ] }, @@ -20,7 +20,7 @@ }, { "dialect": { - "nulls": [ + "missingValues": [ "n/a" ] }, @@ -30,7 +30,7 @@ }, { "dialect": { - "nulls": [ + "missingValues": [ "n/a" ] }, @@ -40,7 +40,7 @@ }, { "dialect": { - "nulls": [ + "missingValues": [ "n/a" ] }, diff --git a/scripts/MoM2003.json b/scripts/MoM2003.json index dac06e1f9..d74d89ca5 100644 --- a/scripts/MoM2003.json +++ b/scripts/MoM2003.json @@ -10,7 +10,7 @@ "resources": [ { "dialect": { - "nulls": [ + "missingValues": [ -999 ], "header_rows": 0 diff --git a/scripts/Palmer2007.json b/scripts/Palmer2007.json index 21744c853..9bd5e4bdb 100644 --- a/scripts/Palmer2007.json +++ b/scripts/Palmer2007.json @@ -24,7 +24,7 @@ }, { "dialect": { - "nulls": [ + "missingValues": [ ".", "" ], diff --git a/scripts/PhytoplankonBiovolume.json b/scripts/PhytoplankonBiovolume.json index 613d1c8cf..89bb70019 100644 --- a/scripts/PhytoplankonBiovolume.json +++ b/scripts/PhytoplankonBiovolume.json @@ -10,7 +10,7 @@ "resources": [ { "dialect": { - "nulls": [ + "missingValues": [ "NA" ] }, @@ -20,7 +20,7 @@ }, { "dialect": { - "nulls": [ + "missingValues": [ "NA" ] }, @@ -30,7 +30,7 @@ }, { "dialect": { - "nulls": [ + "missingValues": [ "NA" ] }, @@ -40,7 +40,7 @@ }, { "dialect": { - "nulls": [ + "missingValues": [ "NA" ] }, @@ -50,7 +50,7 @@ }, { "dialect": { - "nulls": [ + "missingValues": [ "NA" ] }, @@ -60,7 +60,7 @@ }, { "dialect": { - "nulls": [ + "missingValues": [ "NA" ] }, diff --git a/scripts/PortalMammals.json b/scripts/PortalMammals.json index 7bedd7fbc..a0cdccc96 100644 --- a/scripts/PortalMammals.json +++ b/scripts/PortalMammals.json @@ -14,7 +14,7 @@ "dialect": { "contains_pk": "True", "delimiter": ",", - "nulls": [ + "missingValues": [ "", null ] diff --git a/scripts/Ramesh2010.json b/scripts/Ramesh2010.json index 57a725d13..43b3a0448 100644 --- a/scripts/Ramesh2010.json +++ b/scripts/Ramesh2010.json @@ -176,7 +176,7 @@ }, { "dialect": { - "nulls": [ + "missingValues": [ "NA" ] }, diff --git a/scripts/SonoranPerennials.json b/scripts/SonoranPerennials.json index 3adb0e233..9cb7ed11c 100644 --- a/scripts/SonoranPerennials.json +++ b/scripts/SonoranPerennials.json @@ -8,7 +8,7 @@ "resources": [ { "dialect": { - "nulls": [ + "missingValues": [ "NA" ] }, @@ -18,7 +18,7 @@ }, { "dialect": { - "nulls": [ + "missingValues": [ "NA" ] }, @@ -28,7 +28,7 @@ }, { "dialect": { - "nulls": [ + "missingValues": [ "NA" ] }, @@ -38,7 +38,7 @@ }, { "dialect": { - "nulls": [ + "missingValues": [ "NA" ] }, @@ -48,7 +48,7 @@ }, { "dialect": { - "nulls": [ + "missingValues": [ "NA" ] }, @@ -58,7 +58,7 @@ }, { "dialect": { - "nulls": [ + "missingValues": [ "NA" ] }, @@ -68,7 +68,7 @@ }, { "dialect": { - "nulls": [ + "missingValues": [ "NA" ] }, @@ -78,7 +78,7 @@ }, { "dialect": { - "nulls": [ + "missingValues": [ "NA", "nd" ] @@ -89,7 +89,7 @@ }, { "dialect": { - "nulls": [ + "missingValues": [ "NA" ] }, diff --git a/scripts/Steppe_plants_2013.json b/scripts/Steppe_plants_2013.json index b2aafeb15..f7b7749ca 100644 --- a/scripts/Steppe_plants_2013.json +++ b/scripts/Steppe_plants_2013.json @@ -12,7 +12,7 @@ "resources": [ { "dialect": { - "nulls": [ + "missingValues": [ "NA" ] }, @@ -22,7 +22,7 @@ }, { "dialect": { - "nulls": [ + "missingValues": [ "NA" ] }, @@ -32,7 +32,7 @@ }, { "dialect": { - "nulls": [ + "missingValues": [ "NA" ] }, @@ -42,7 +42,7 @@ }, { "dialect": { - "nulls": [ + "missingValues": [ "NA" ] }, @@ -52,7 +52,7 @@ }, { "dialect": { - "nulls": [ + "missingValues": [ "NA" ] }, @@ -62,7 +62,7 @@ }, { "dialect": { - "nulls": [ + "missingValues": [ "NA" ] }, @@ -72,7 +72,7 @@ }, { "dialect": { - "nulls": [ + "missingValues": [ "NA" ] }, @@ -82,7 +82,7 @@ }, { "dialect": { - "nulls": [ + "missingValues": [ "NA" ] }, diff --git a/scripts/TreeWesternGhats.json b/scripts/TreeWesternGhats.json index 3c58c8141..36fbccd58 100644 --- a/scripts/TreeWesternGhats.json +++ b/scripts/TreeWesternGhats.json @@ -10,7 +10,7 @@ "resources": [ { "dialect": { - "nulls": [ + "missingValues": [ "NA" ] }, @@ -20,7 +20,7 @@ }, { "dialect": { - "nulls": [ + "missingValues": [ "NA" ] }, diff --git a/scripts/Woods2009.json b/scripts/Woods2009.json index b2fae9a72..c543a3e73 100644 --- a/scripts/Woods2009.json +++ b/scripts/Woods2009.json @@ -12,7 +12,7 @@ "resources": [ { "dialect": { - "nulls": [ + "missingValues": [ "NA" ] }, @@ -22,7 +22,7 @@ }, { "dialect": { - "nulls": [ + "missingValues": [ "NA" ] }, @@ -44,7 +44,7 @@ }, { "dialect": { - "nulls": [ + "missingValues": [ "NA", "" ] @@ -55,7 +55,7 @@ }, { "dialect": { - "nulls": [ + "missingValues": [ "NA" ] }, diff --git a/test/test_retriever.py b/test/test_retriever.py index 6efbb5f9d..dba0279bc 100644 --- a/test/test_retriever.py +++ b/test/test_retriever.py @@ -359,7 +359,7 @@ def test_add_dialect(): table = {} table['dialect'] = {} table_dict = {} - table['dialect']['nulls'] = '\0' + table['dialect']['missingValues'] = '\0' table['dialect']['delimiter'] = '\t' table['dialect']['dummy_key'] = 'dummy_value'