Skip to content

Commit

Permalink
Update images
Browse files Browse the repository at this point in the history
  • Loading branch information
@Erotemic
Erotemic committed Sep 23, 2023
1 parent 9a24226 commit fae2795
Show file tree
Hide file tree
Showing 5 changed files with 88 additions and 49 deletions.
18 changes: 14 additions & 4 deletions README.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,8 @@ Recent Updates
Check back for updates, but because this is a personal project, it might take
some time for it to fully drop.

* 2023-09-22 - 123 new images.
* 2023-08-22 - 182 new images.
* 2023-07-01 - Another batch of 300 photos. I also realized that if I could ID which dog made which poop, I could do a longiturdinal study.
* 2023-04-16 - More ground based photos. One "after" photo contains a positive example I didn't see in the background.
* 2023-03-11 - 305 new images. Many of these images are taken from a close up ground angle. I will continue to collect more in this way.
Expand Down Expand Up @@ -245,6 +247,11 @@ registration via the SIFT+RANSAC algorithm.
+-------------+----------+---------------------+-----------------------+
| 2023-07-01 | 4594 | ~1662 | 1154 |
+-------------+----------+---------------------+-----------------------+
| 2023-08-22 | 4776 | ~1723 | 1197 |
+-------------+----------+---------------------+-----------------------+
| 2023-09-22 | 4899 | ~1764 | 1232 |
+-------------+----------+---------------------+-----------------------+



Annotation Process
Expand Down Expand Up @@ -296,7 +303,7 @@ In addition to these licenses please:

Otherwise the data is free to use commercially or otherwise.

The URL that can be viewed in a web browser: https://ipfs.io/ipfs/bafybeiflkm37altah2ey2jxko7kngquwfugyo4cl36y7xjf7o2lbrgucbi
The URL that can be viewed in a web browser: https://ipfs.io/ipfs/bafybeiczi4pn4na2iw7c66bpbf5rdr3ua3grp2qvjgrmnuzqabjjim4o2q

Current IPFS addresses for the top-level dataset filesystem are:

Expand Down Expand Up @@ -331,10 +338,13 @@ Current IPFS addresses for the top-level dataset filesystem are:
bafybeibx2oarr3liqrda4hd7xlw643vbd5nxff2b44blzccw7ekw6gbwv4 - shitspotter_dvc/assets/poop-2023-01-01-T171030/
bafybeibky4jj4hhmlwuifx52fjdurseqzkmwpp4derwqvf5lo2vakzrtoe - shitspotter_dvc/assets/poop-2023-03-11-T165018/
bafybeifj7uidepqz2wbumajacy2oacn7c7cuh6zwnduovn4xyszdpiodoe - shitspotter_dvc/assets/poop-2023-04-16-T175739/
bafybeihhbwe6mtkts7335e2wdr3p4mo5impx3niqbcavvqh3l3rknpbuti - shitspotter_dvc/assets/poop-2023-07-01-T160318
bafybeihhbwe6mtkts7335e2wdr3p4mo5impx3niqbcavvqh3l3rknpbuti - shitspotter_dvc/assets/poop-2023-07-01-T160318/
bafybeiez6f2nwubarmduko73uclgitsaagvdov4s5oexcwltw5dosjhq4m - shitspotter_dvc/assets/poop-2023-08-22-T202656/
bafybeihurilrwce7rxr7o3iqdf227o74cfk23ilv2nleoj5hd6wx5iapz4 - shitspotter_dvc/assets/poop-2023-09-22-T180825
bafybeibsljcqg2sd2rpho2xd4z6fimzxwyvugmgqasoxseq7qcomnpku5q - shitspotter_dvc/assets
bafybeiflkm37altah2ey2jxko7kngquwfugyo4cl36y7xjf7o2lbrgucbi - shitspotter_dvc
bafybeidtzbnrua46vjx36ragh7z5ouyodqazh4cxyetdxxtimvk7yi46qu - shitspotter_dvc/assets
bafybeieahblb6aafomi72gnheu3ihom7nobdad4t6jcrrwhd5eb3wxkrgy - shitspotter_dvc
Expand Down
89 changes: 47 additions & 42 deletions dev/remote_from_phone.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -230,6 +230,7 @@ def main():
# Import to make sure they are installed
import shitspotter # NOQA
import xdev # NOQA
import pickle

cache_dpath = ub.Path.appdir('shitspotter/transfer_session').ensuredir()
lock_fpath = cache_dpath / 'transfering.lock'
Expand All @@ -240,9 +241,13 @@ def main():
'Needs to implement resume or cleanup dirty state')
lock_fpath.touch()

new_dpath, needs_transfer_infos = prepare_phone_transfer(config)
try:
new_dpath, needs_transfer_infos = prepare_phone_transfer(config)
except Exception:
print('FIXME: delete cache?')
cache_dpath.delete()
raise

import pickle
prepared_transfer_fpath.write_bytes(pickle.dumps({
'new_dpath': new_dpath,
'needs_transfer_infos': needs_transfer_infos,
Expand Down Expand Up @@ -367,6 +372,8 @@ def safe_copy(src, dst):
eager_copy_jobs = [d for d in copy_jobs if not d['dst'].exists()]
print(f'# Needs Copy {len(eager_copy_jobs)} / {len(copy_jobs)}')

# Could use kwutil CopyManager

for copy_job in ub.ProgIter(copy_jobs, desc='submit jobs'):
src, dst = copy_job['src'], copy_job['dst']
job = jobs.submit(safe_copy, src, dst)
Expand Down Expand Up @@ -431,7 +438,7 @@ def finalize_transfer(new_dpath):
# print('Next step is to run the matching script: `python -m shitspotter.matching autofind_pair_hueristic`')
# print('Next step is to run the plots script: `python -m shitspotter.plots update_analysis_plots`')
print('')
print('Then repin the updated dataset to IPFS')
print('# Then repin the updated dataset to IPFS')

shitspotter_dvc_dpath = coco_fpath.parent
command = ub.codeblock(
Expand All @@ -456,7 +463,7 @@ def finalize_transfer(new_dpath):
echo "NEW_ROOT_CID=$NEW_ROOT_CID"
# Add it to the CID revisions:
echo "$NEW_ROOT_CID" >> $HOME/code/shitspotter/shitspotter/cid_revisions.txt
echo "$NEW_ROOT_CID" >> "$HOME"/code/shitspotter/shitspotter/cid_revisions.txt
echo "
Then on MOJO run:
Expand All @@ -466,6 +473,7 @@ def finalize_transfer(new_dpath):
ipfs pin add --progress $NEW_ROOT_CID
"
# Also see: ~/code/shitspotter/dev/sync_shit.sh
'''
)
# TODO: is there another pinning service that wont flake on us?
Expand All @@ -478,44 +486,41 @@ def finalize_transfer(new_dpath):

# dpath = ub.Path(shitspotter.__file__).parent
# cid_revisions_fpath = dpath / 'cid_revisions.txt'
command = ub.codeblock(
'''
OLD EXAMPLES:
echo "QmNj2MbeL183GtPoGkFv569vMY8nupUVGEVvvvqhjoAATG" >> /home/joncrall/code/shitspotter/shitspotter/cid_revisions.txt
echo "QmaPPoPs7wXXkBgJeffVm49rd63ZtZw5GrhvQQbYrUbrYL" >> /home/joncrall/code/shitspotter/shitspotter/cid_revisions.txt
echo "QmaSfRtzXDCiqyfmZuH6NEy2HBr7radiJNhmSjiETihoh6" >> /home/joncrall/code/shitspotter/shitspotter/cid_revisions.txt
echo "QmPptXKFKi6oTJL3VeCNy5Apk8MJsHhCAAwVmegHhuRY83" >> /home/joncrall/code/shitspotter/shitspotter/cid_revisions.txt
echo "QmfStoay5rjeHMEDiyuGsreXNHsyiS5kVaexSM2fov216j" >> /home/joncrall/code/shitspotter/shitspotter/cid_revisions.txt
echo "bafybeihgex2fj4ethxnfmiivasw5wqsbt2pdjswu3yr554rewm6myrkq4a" >> /home/joncrall/code/shitspotter/shitspotter/cid_revisions.txt
echo "bafybeihltrtb4xncqvfbipdwnlxsrxmeb4df7xmoqpjatg7jxrl3lqqk6y" >> /home/joncrall/code/shitspotter/shitspotter/cid_revisions.txt
echo "bafybeihi7v7sgnxb2y57ie2dr7oobigsn5fqiwxwq56sdpmzo5on7a2xwe" >> /home/joncrall/code/shitspotter/shitspotter/cid_revisions.txt
echo "bafybeiedk6bu2qpl4snlu3jmtri4b2sf476tgj5kdg2ztxtm7bd6ftzqyy" >> /home/joncrall/code/shitspotter/shitspotter/cid_revisions.txt
Then on mojo:
NEW_ROOT_CID=bafybeihi7v7sgnxb2y57ie2dr7oobigsn5fqiwxwq56sdpmzo5on7a2xwe
NEW_ROOT_CID=bafybeiedk6bu2qpl4snlu3jmtri4b2sf476tgj5kdg2ztxtm7bd6ftzqyy
DATE=$(date +"%Y-%m-%d")
ipfs pin add --progress "${NEW_ROOT_CID}"
# Also, we should pin the CID on a pinning service
ipfs pin remote add --service=web3.storage.erotemic --name=shitspotter-dvc-$DATE ${NEW_ROOT_CID} --background
ipfs pin remote ls --service=web3.storage.erotemic --cid=${NEW_ROOT_CID} --status=queued,pinning,pinned,failed
# e.g.
ipfs pin add bafybeihgex2fj4ethxnfmiivasw5wqsbt2pdjswu3yr554rewm6myrkq4a --progress
ipfs pin add QmfStoay5rjeHMEDiyuGsreXNHsyiS5kVaexSM2fov216j --progress
ipfs pin remote add --service=web3.storage.erotemic --name=shitspotter-dvc-2022-06-08 bafybeihgex2fj4ethxnfmiivasw5wqsbt2pdjswu3yr554rewm6myrkq4a --background
ipfs pin remote ls --service=web3.storage.erotemic --cid=QmYftzG6enTebF2f143KeHiPiJGs66LJf3jT1fNYAiqQvq --status=queued,pinning,pinned,failed
Also see: ~/code/shitspotter/dev/sync_shit.sh
'''
)
print(command)
# command = ub.codeblock(
# '''

# OLD EXAMPLES:
# echo "QmNj2MbeL183GtPoGkFv569vMY8nupUVGEVvvvqhjoAATG" >> /home/joncrall/code/shitspotter/shitspotter/cid_revisions.txt
# echo "QmaPPoPs7wXXkBgJeffVm49rd63ZtZw5GrhvQQbYrUbrYL" >> /home/joncrall/code/shitspotter/shitspotter/cid_revisions.txt
# echo "QmaSfRtzXDCiqyfmZuH6NEy2HBr7radiJNhmSjiETihoh6" >> /home/joncrall/code/shitspotter/shitspotter/cid_revisions.txt
# echo "QmPptXKFKi6oTJL3VeCNy5Apk8MJsHhCAAwVmegHhuRY83" >> /home/joncrall/code/shitspotter/shitspotter/cid_revisions.txt
# echo "QmfStoay5rjeHMEDiyuGsreXNHsyiS5kVaexSM2fov216j" >> /home/joncrall/code/shitspotter/shitspotter/cid_revisions.txt
# echo "bafybeihgex2fj4ethxnfmiivasw5wqsbt2pdjswu3yr554rewm6myrkq4a" >> /home/joncrall/code/shitspotter/shitspotter/cid_revisions.txt
# echo "bafybeihltrtb4xncqvfbipdwnlxsrxmeb4df7xmoqpjatg7jxrl3lqqk6y" >> /home/joncrall/code/shitspotter/shitspotter/cid_revisions.txt
# echo "bafybeihi7v7sgnxb2y57ie2dr7oobigsn5fqiwxwq56sdpmzo5on7a2xwe" >> /home/joncrall/code/shitspotter/shitspotter/cid_revisions.txt
# echo "bafybeiedk6bu2qpl4snlu3jmtri4b2sf476tgj5kdg2ztxtm7bd6ftzqyy" >> /home/joncrall/code/shitspotter/shitspotter/cid_revisions.txt

# Then on mojo:

# # Note: define NEW_ROOT_CID=
# DATE=$(date +"%Y-%m-%d")

# ipfs pin add --progress "${NEW_ROOT_CID}"

# # Also, we should pin the CID on a pinning service
# ipfs pin remote add --service=web3.storage.erotemic --name="shitspotter-dvc-$DATE" "${NEW_ROOT_CID}" --background
# ipfs pin remote ls --service=web3.storage.erotemic --cid="${NEW_ROOT_CID}" --status=queued,pinning,pinned,failed

# # e.g.
# ipfs pin add bafybeihgex2fj4ethxnfmiivasw5wqsbt2pdjswu3yr554rewm6myrkq4a --progress
# ipfs pin add QmfStoay5rjeHMEDiyuGsreXNHsyiS5kVaexSM2fov216j --progress
# ipfs pin remote add --service=web3.storage.erotemic --name=shitspotter-dvc-2022-06-08 bafybeihgex2fj4ethxnfmiivasw5wqsbt2pdjswu3yr554rewm6myrkq4a --background
# ipfs pin remote ls --service=web3.storage.erotemic --cid=QmYftzG6enTebF2f143KeHiPiJGs66LJf3jT1fNYAiqQvq --status=queued,pinning,pinned,failed

# Also see: ~/code/shitspotter/dev/sync_shit.sh
# '''
# )
# print(command)

# Try to programatically figure out what the CID was
r"""
Expand Down
3 changes: 3 additions & 0 deletions shitspotter/cid_revisions.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,3 +14,6 @@ bafybeicjvjt2abdj7e5mpwq27itxi2u6lzcegl5dgw6nqe22363vmdsnru
bafybeic2ehnqled363zqimtbqbonagw6atgsyst5cqbm3wec6cg3te5ala

bafybeiflkm37altah2ey2jxko7kngquwfugyo4cl36y7xjf7o2lbrgucbi
bafybeiczi4pn4na2iw7c66bpbf5rdr3ua3grp2qvjgrmnuzqabjjim4o2q
bafybeiczi4pn4na2iw7c66bpbf5rdr3ua3grp2qvjgrmnuzqabjjim4o2q
bafybeieahblb6aafomi72gnheu3ihom7nobdad4t6jcrrwhd5eb3wxkrgy
23 changes: 22 additions & 1 deletion shitspotter/matching.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -261,7 +261,28 @@ def matchable_image(gid):
print('num_pairs_ideal = {!r}'.format(num_pairs_ideal))
print('num_triples_ideal = {!r}'.format(num_triples_ideal))

print(f'total_images = {len(coco_dset.imgs)}')
total_imgs = len(coco_dset.imgs)
print(f'total_images = {total_imgs}')

if 1:
import datetime as datetime_mod
today = datetime_mod.datetime.now().date()
row = {
'Date': today.isoformat(),
'# Images': total_imgs,
'# Estimated Groups': total_estimated_number_of_tups,
'# Registered Groups': total_matchable_tups,
}
print('New row for README')
print('| {:<12s}| {:<8s} | {:<18s} | {:<22s}|'.format(*list(row.keys())))
print('+=============+==========+=====================+=======================+')
print('| {:<12s}| {:<7d} | ~{:<17d} | {:<22d}|'.format(*list(row.values())))
print('+-------------+----------+---------------------+-----------------------+')
# import tabulate
# import pandas as pd
# df = pd.DataFrame([row])
# print(tabulate.tabulate(df.values, headers=df.columns, tablefmt='rst'))

# print('total_estimated_number_of_pairs = {!r}'.format(total_estimated_number_of_pairs))
# if 0:
# import kwplot
Expand Down
4 changes: 2 additions & 2 deletions shitspotter/plots.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -474,14 +474,14 @@ def demo_warp(coco_dset, gid1, gid2):
rchip1_rgb, sf_info1 = kwimage.imresize(imdata1, max_dim=maxdim, return_info=True)
rchip2_rgb, sf_info2 = kwimage.imresize(imdata2, max_dim=maxdim, return_info=True)

undo_scale = kwimage.Affine.coerce(sf_info1).inv()
# rchip2_rgb, sf_info2 = kwimage.imresize(imdata2, max_dim=512, return_info=True)
# rchip1_rgb, sf_info1 = kwimage.imresize(imdata1, max_dim=800, return_info=True)
# rchip2_rgb, sf_info2 = kwimage.imresize(imdata2, max_dim=800, return_info=True)
rchip1_rgb, sf_info1 = kwimage.imresize(imdata1, max_dim=maxdim, return_info=True)
rchip2_rgb, sf_info2 = kwimage.imresize(imdata2, max_dim=maxdim, return_info=True)

undo_scale = kwimage.Affine.coerce(sf_info1).inv()
undo_scale = kwimage.Affine.coerce(ub.udict(sf_info1) - {'dsize'}).inv()
# undo_scale = kwimage.Affine.coerce(sf_info1).inv()

# import kwplot
# kwplot.autompl()
Expand Down

0 comments on commit fae2795

Please sign in to comment.