diff --git a/README.rst b/README.rst index 6847d16..4fd0718 100644 --- a/README.rst +++ b/README.rst @@ -59,6 +59,8 @@ Recent Updates Check back for updates, but because this is a personal project, it might take some time for it to fully drop. +* 2023-09-22 - 123 new images. +* 2023-08-22 - 182 new images. * 2023-07-01 - Another batch of 300 photos. I also realized that if I could ID which dog made which poop, I could do a longiturdinal study. * 2023-04-16 - More ground based photos. One "after" photo contains a positive example I didn't see in the background. * 2023-03-11 - 305 new images. Many of these images are taken from a close up ground angle. I will continue to collect more in this way. @@ -245,6 +247,11 @@ registration via the SIFT+RANSAC algorithm. +-------------+----------+---------------------+-----------------------+ | 2023-07-01 | 4594 | ~1662 | 1154 | +-------------+----------+---------------------+-----------------------+ +| 2023-08-22 | 4776 | ~1723 | 1197 | ++-------------+----------+---------------------+-----------------------+ +| 2023-09-22 | 4899 | ~1764 | 1232 | ++-------------+----------+---------------------+-----------------------+ + Annotation Process @@ -296,7 +303,7 @@ In addition to these licenses please: Otherwise the data is free to use commercially or otherwise. -The URL that can be viewed in a web browser: https://ipfs.io/ipfs/bafybeiflkm37altah2ey2jxko7kngquwfugyo4cl36y7xjf7o2lbrgucbi +The URL that can be viewed in a web browser: https://ipfs.io/ipfs/bafybeiczi4pn4na2iw7c66bpbf5rdr3ua3grp2qvjgrmnuzqabjjim4o2q Current IPFS addresses for the top-level dataset filesystem are: @@ -331,10 +338,13 @@ Current IPFS addresses for the top-level dataset filesystem are: bafybeibx2oarr3liqrda4hd7xlw643vbd5nxff2b44blzccw7ekw6gbwv4 - shitspotter_dvc/assets/poop-2023-01-01-T171030/ bafybeibky4jj4hhmlwuifx52fjdurseqzkmwpp4derwqvf5lo2vakzrtoe - shitspotter_dvc/assets/poop-2023-03-11-T165018/ bafybeifj7uidepqz2wbumajacy2oacn7c7cuh6zwnduovn4xyszdpiodoe - shitspotter_dvc/assets/poop-2023-04-16-T175739/ - bafybeihhbwe6mtkts7335e2wdr3p4mo5impx3niqbcavvqh3l3rknpbuti - shitspotter_dvc/assets/poop-2023-07-01-T160318 + bafybeihhbwe6mtkts7335e2wdr3p4mo5impx3niqbcavvqh3l3rknpbuti - shitspotter_dvc/assets/poop-2023-07-01-T160318/ + bafybeiez6f2nwubarmduko73uclgitsaagvdov4s5oexcwltw5dosjhq4m - shitspotter_dvc/assets/poop-2023-08-22-T202656/ + bafybeihurilrwce7rxr7o3iqdf227o74cfk23ilv2nleoj5hd6wx5iapz4 - shitspotter_dvc/assets/poop-2023-09-22-T180825 + - bafybeibsljcqg2sd2rpho2xd4z6fimzxwyvugmgqasoxseq7qcomnpku5q - shitspotter_dvc/assets - bafybeiflkm37altah2ey2jxko7kngquwfugyo4cl36y7xjf7o2lbrgucbi - shitspotter_dvc + bafybeidtzbnrua46vjx36ragh7z5ouyodqazh4cxyetdxxtimvk7yi46qu - shitspotter_dvc/assets + bafybeieahblb6aafomi72gnheu3ihom7nobdad4t6jcrrwhd5eb3wxkrgy - shitspotter_dvc diff --git a/dev/remote_from_phone.py b/dev/remote_from_phone.py index 604185b..e44d29e 100644 --- a/dev/remote_from_phone.py +++ b/dev/remote_from_phone.py @@ -230,6 +230,7 @@ def main(): # Import to make sure they are installed import shitspotter # NOQA import xdev # NOQA + import pickle cache_dpath = ub.Path.appdir('shitspotter/transfer_session').ensuredir() lock_fpath = cache_dpath / 'transfering.lock' @@ -240,9 +241,13 @@ def main(): 'Needs to implement resume or cleanup dirty state') lock_fpath.touch() - new_dpath, needs_transfer_infos = prepare_phone_transfer(config) + try: + new_dpath, needs_transfer_infos = prepare_phone_transfer(config) + except Exception: + print('FIXME: delete cache?') + cache_dpath.delete() + raise - import pickle prepared_transfer_fpath.write_bytes(pickle.dumps({ 'new_dpath': new_dpath, 'needs_transfer_infos': needs_transfer_infos, @@ -367,6 +372,8 @@ def safe_copy(src, dst): eager_copy_jobs = [d for d in copy_jobs if not d['dst'].exists()] print(f'# Needs Copy {len(eager_copy_jobs)} / {len(copy_jobs)}') + # Could use kwutil CopyManager + for copy_job in ub.ProgIter(copy_jobs, desc='submit jobs'): src, dst = copy_job['src'], copy_job['dst'] job = jobs.submit(safe_copy, src, dst) @@ -431,7 +438,7 @@ def finalize_transfer(new_dpath): # print('Next step is to run the matching script: `python -m shitspotter.matching autofind_pair_hueristic`') # print('Next step is to run the plots script: `python -m shitspotter.plots update_analysis_plots`') print('') - print('Then repin the updated dataset to IPFS') + print('# Then repin the updated dataset to IPFS') shitspotter_dvc_dpath = coco_fpath.parent command = ub.codeblock( @@ -456,7 +463,7 @@ def finalize_transfer(new_dpath): echo "NEW_ROOT_CID=$NEW_ROOT_CID" # Add it to the CID revisions: - echo "$NEW_ROOT_CID" >> $HOME/code/shitspotter/shitspotter/cid_revisions.txt + echo "$NEW_ROOT_CID" >> "$HOME"/code/shitspotter/shitspotter/cid_revisions.txt echo " Then on MOJO run: @@ -466,6 +473,7 @@ def finalize_transfer(new_dpath): ipfs pin add --progress $NEW_ROOT_CID " + # Also see: ~/code/shitspotter/dev/sync_shit.sh ''' ) # TODO: is there another pinning service that wont flake on us? @@ -478,44 +486,41 @@ def finalize_transfer(new_dpath): # dpath = ub.Path(shitspotter.__file__).parent # cid_revisions_fpath = dpath / 'cid_revisions.txt' - command = ub.codeblock( - ''' - - OLD EXAMPLES: - - echo "QmNj2MbeL183GtPoGkFv569vMY8nupUVGEVvvvqhjoAATG" >> /home/joncrall/code/shitspotter/shitspotter/cid_revisions.txt - echo "QmaPPoPs7wXXkBgJeffVm49rd63ZtZw5GrhvQQbYrUbrYL" >> /home/joncrall/code/shitspotter/shitspotter/cid_revisions.txt - echo "QmaSfRtzXDCiqyfmZuH6NEy2HBr7radiJNhmSjiETihoh6" >> /home/joncrall/code/shitspotter/shitspotter/cid_revisions.txt - echo "QmPptXKFKi6oTJL3VeCNy5Apk8MJsHhCAAwVmegHhuRY83" >> /home/joncrall/code/shitspotter/shitspotter/cid_revisions.txt - echo "QmfStoay5rjeHMEDiyuGsreXNHsyiS5kVaexSM2fov216j" >> /home/joncrall/code/shitspotter/shitspotter/cid_revisions.txt - echo "bafybeihgex2fj4ethxnfmiivasw5wqsbt2pdjswu3yr554rewm6myrkq4a" >> /home/joncrall/code/shitspotter/shitspotter/cid_revisions.txt - echo "bafybeihltrtb4xncqvfbipdwnlxsrxmeb4df7xmoqpjatg7jxrl3lqqk6y" >> /home/joncrall/code/shitspotter/shitspotter/cid_revisions.txt - echo "bafybeihi7v7sgnxb2y57ie2dr7oobigsn5fqiwxwq56sdpmzo5on7a2xwe" >> /home/joncrall/code/shitspotter/shitspotter/cid_revisions.txt - echo "bafybeiedk6bu2qpl4snlu3jmtri4b2sf476tgj5kdg2ztxtm7bd6ftzqyy" >> /home/joncrall/code/shitspotter/shitspotter/cid_revisions.txt - - - Then on mojo: - - NEW_ROOT_CID=bafybeihi7v7sgnxb2y57ie2dr7oobigsn5fqiwxwq56sdpmzo5on7a2xwe - NEW_ROOT_CID=bafybeiedk6bu2qpl4snlu3jmtri4b2sf476tgj5kdg2ztxtm7bd6ftzqyy - DATE=$(date +"%Y-%m-%d") - - ipfs pin add --progress "${NEW_ROOT_CID}" - - # Also, we should pin the CID on a pinning service - ipfs pin remote add --service=web3.storage.erotemic --name=shitspotter-dvc-$DATE ${NEW_ROOT_CID} --background - ipfs pin remote ls --service=web3.storage.erotemic --cid=${NEW_ROOT_CID} --status=queued,pinning,pinned,failed - - # e.g. - ipfs pin add bafybeihgex2fj4ethxnfmiivasw5wqsbt2pdjswu3yr554rewm6myrkq4a --progress - ipfs pin add QmfStoay5rjeHMEDiyuGsreXNHsyiS5kVaexSM2fov216j --progress - ipfs pin remote add --service=web3.storage.erotemic --name=shitspotter-dvc-2022-06-08 bafybeihgex2fj4ethxnfmiivasw5wqsbt2pdjswu3yr554rewm6myrkq4a --background - ipfs pin remote ls --service=web3.storage.erotemic --cid=QmYftzG6enTebF2f143KeHiPiJGs66LJf3jT1fNYAiqQvq --status=queued,pinning,pinned,failed - - Also see: ~/code/shitspotter/dev/sync_shit.sh - ''' - ) - print(command) + # command = ub.codeblock( + # ''' + + # OLD EXAMPLES: + # echo "QmNj2MbeL183GtPoGkFv569vMY8nupUVGEVvvvqhjoAATG" >> /home/joncrall/code/shitspotter/shitspotter/cid_revisions.txt + # echo "QmaPPoPs7wXXkBgJeffVm49rd63ZtZw5GrhvQQbYrUbrYL" >> /home/joncrall/code/shitspotter/shitspotter/cid_revisions.txt + # echo "QmaSfRtzXDCiqyfmZuH6NEy2HBr7radiJNhmSjiETihoh6" >> /home/joncrall/code/shitspotter/shitspotter/cid_revisions.txt + # echo "QmPptXKFKi6oTJL3VeCNy5Apk8MJsHhCAAwVmegHhuRY83" >> /home/joncrall/code/shitspotter/shitspotter/cid_revisions.txt + # echo "QmfStoay5rjeHMEDiyuGsreXNHsyiS5kVaexSM2fov216j" >> /home/joncrall/code/shitspotter/shitspotter/cid_revisions.txt + # echo "bafybeihgex2fj4ethxnfmiivasw5wqsbt2pdjswu3yr554rewm6myrkq4a" >> /home/joncrall/code/shitspotter/shitspotter/cid_revisions.txt + # echo "bafybeihltrtb4xncqvfbipdwnlxsrxmeb4df7xmoqpjatg7jxrl3lqqk6y" >> /home/joncrall/code/shitspotter/shitspotter/cid_revisions.txt + # echo "bafybeihi7v7sgnxb2y57ie2dr7oobigsn5fqiwxwq56sdpmzo5on7a2xwe" >> /home/joncrall/code/shitspotter/shitspotter/cid_revisions.txt + # echo "bafybeiedk6bu2qpl4snlu3jmtri4b2sf476tgj5kdg2ztxtm7bd6ftzqyy" >> /home/joncrall/code/shitspotter/shitspotter/cid_revisions.txt + + # Then on mojo: + + # # Note: define NEW_ROOT_CID= + # DATE=$(date +"%Y-%m-%d") + + # ipfs pin add --progress "${NEW_ROOT_CID}" + + # # Also, we should pin the CID on a pinning service + # ipfs pin remote add --service=web3.storage.erotemic --name="shitspotter-dvc-$DATE" "${NEW_ROOT_CID}" --background + # ipfs pin remote ls --service=web3.storage.erotemic --cid="${NEW_ROOT_CID}" --status=queued,pinning,pinned,failed + + # # e.g. + # ipfs pin add bafybeihgex2fj4ethxnfmiivasw5wqsbt2pdjswu3yr554rewm6myrkq4a --progress + # ipfs pin add QmfStoay5rjeHMEDiyuGsreXNHsyiS5kVaexSM2fov216j --progress + # ipfs pin remote add --service=web3.storage.erotemic --name=shitspotter-dvc-2022-06-08 bafybeihgex2fj4ethxnfmiivasw5wqsbt2pdjswu3yr554rewm6myrkq4a --background + # ipfs pin remote ls --service=web3.storage.erotemic --cid=QmYftzG6enTebF2f143KeHiPiJGs66LJf3jT1fNYAiqQvq --status=queued,pinning,pinned,failed + + # Also see: ~/code/shitspotter/dev/sync_shit.sh + # ''' + # ) + # print(command) # Try to programatically figure out what the CID was r""" diff --git a/shitspotter/cid_revisions.txt b/shitspotter/cid_revisions.txt index ac3ca83..d5a640a 100644 --- a/shitspotter/cid_revisions.txt +++ b/shitspotter/cid_revisions.txt @@ -14,3 +14,6 @@ bafybeicjvjt2abdj7e5mpwq27itxi2u6lzcegl5dgw6nqe22363vmdsnru bafybeic2ehnqled363zqimtbqbonagw6atgsyst5cqbm3wec6cg3te5ala bafybeiflkm37altah2ey2jxko7kngquwfugyo4cl36y7xjf7o2lbrgucbi +bafybeiczi4pn4na2iw7c66bpbf5rdr3ua3grp2qvjgrmnuzqabjjim4o2q +bafybeiczi4pn4na2iw7c66bpbf5rdr3ua3grp2qvjgrmnuzqabjjim4o2q +bafybeieahblb6aafomi72gnheu3ihom7nobdad4t6jcrrwhd5eb3wxkrgy diff --git a/shitspotter/matching.py b/shitspotter/matching.py index a9dc5d4..122d9d6 100644 --- a/shitspotter/matching.py +++ b/shitspotter/matching.py @@ -261,7 +261,28 @@ def matchable_image(gid): print('num_pairs_ideal = {!r}'.format(num_pairs_ideal)) print('num_triples_ideal = {!r}'.format(num_triples_ideal)) - print(f'total_images = {len(coco_dset.imgs)}') + total_imgs = len(coco_dset.imgs) + print(f'total_images = {total_imgs}') + + if 1: + import datetime as datetime_mod + today = datetime_mod.datetime.now().date() + row = { + 'Date': today.isoformat(), + '# Images': total_imgs, + '# Estimated Groups': total_estimated_number_of_tups, + '# Registered Groups': total_matchable_tups, + } + print('New row for README') + print('| {:<12s}| {:<8s} | {:<18s} | {:<22s}|'.format(*list(row.keys()))) + print('+=============+==========+=====================+=======================+') + print('| {:<12s}| {:<7d} | ~{:<17d} | {:<22d}|'.format(*list(row.values()))) + print('+-------------+----------+---------------------+-----------------------+') + # import tabulate + # import pandas as pd + # df = pd.DataFrame([row]) + # print(tabulate.tabulate(df.values, headers=df.columns, tablefmt='rst')) + # print('total_estimated_number_of_pairs = {!r}'.format(total_estimated_number_of_pairs)) # if 0: # import kwplot diff --git a/shitspotter/plots.py b/shitspotter/plots.py index 50e8cdb..c8bc904 100644 --- a/shitspotter/plots.py +++ b/shitspotter/plots.py @@ -474,14 +474,14 @@ def demo_warp(coco_dset, gid1, gid2): rchip1_rgb, sf_info1 = kwimage.imresize(imdata1, max_dim=maxdim, return_info=True) rchip2_rgb, sf_info2 = kwimage.imresize(imdata2, max_dim=maxdim, return_info=True) - undo_scale = kwimage.Affine.coerce(sf_info1).inv() # rchip2_rgb, sf_info2 = kwimage.imresize(imdata2, max_dim=512, return_info=True) # rchip1_rgb, sf_info1 = kwimage.imresize(imdata1, max_dim=800, return_info=True) # rchip2_rgb, sf_info2 = kwimage.imresize(imdata2, max_dim=800, return_info=True) rchip1_rgb, sf_info1 = kwimage.imresize(imdata1, max_dim=maxdim, return_info=True) rchip2_rgb, sf_info2 = kwimage.imresize(imdata2, max_dim=maxdim, return_info=True) - undo_scale = kwimage.Affine.coerce(sf_info1).inv() + undo_scale = kwimage.Affine.coerce(ub.udict(sf_info1) - {'dsize'}).inv() + # undo_scale = kwimage.Affine.coerce(sf_info1).inv() # import kwplot # kwplot.autompl()