From 9eb3ca37a3f43711372712613624f276f3177627 Mon Sep 17 00:00:00 2001 From: drtpotter Date: Fri, 1 Mar 2019 14:49:28 +0800 Subject: [PATCH 01/19] Added NGAS plugin with simple file upload/download functionality --- fabfile/APPspecific.py | 8 +- .../server/spaces/ngas/.idea/encodings.xml | 4 + .../ngas/.idea/libraries/R_User_Library.xml | 6 + pyvospace/server/spaces/ngas/.idea/misc.xml | 4 + .../server/spaces/ngas/.idea/modules.xml | 8 + pyvospace/server/spaces/ngas/.idea/ngas.iml | 14 + pyvospace/server/spaces/ngas/.idea/vcs.xml | 6 + .../server/spaces/ngas/.idea/workspace.xml | 106 ++++++ pyvospace/server/spaces/ngas/__init__.py | 1 + pyvospace/server/spaces/ngas/auth.py | 223 +++++++++++ .../server/spaces/ngas/space/__init__.py | 0 .../server/spaces/ngas/space/__main__.py | 62 +++ .../server/spaces/ngas/space/ngas_space.py | 216 +++++++++++ .../server/spaces/ngas/storage/__init__.py | 0 .../server/spaces/ngas/storage/__main__.py | 61 +++ .../spaces/ngas/storage/ngas_storage.py | 343 +++++++++++++++++ pyvospace/server/spaces/ngas/utils.py | 344 +++++++++++++++++ scripts/start_container_db.sh | 4 + scripts/start_ngas.sh | 3 + scripts/start_ngas_space.sh | 6 + scripts/start_ngas_storage.sh | 6 + setup.py | 6 +- test/.DS_Store | Bin 0 -> 6148 bytes test/test_ngas/__init__.py | 0 test/test_ngas/test_abort.py | 142 +++++++ test/test_ngas/test_base.py | 353 ++++++++++++++++++ test/{ => test_ngas}/test_copy_move.py | 0 test/{ => test_ngas}/test_create.py | 0 test/test_ngas/test_push_pull.py | 325 ++++++++++++++++ test/test_ngas/test_simple.py | 145 +++++++ test/test_ngas/test_vo_ngas.ini | 21 ++ test/test_posix/__init__.py | 0 test/{ => test_posix}/test_abort.py | 0 test/{ => test_posix}/test_base.py | 2 +- test/test_posix/test_copy_move.py | 325 ++++++++++++++++ test/test_posix/test_create.py | 267 +++++++++++++ test/{ => test_posix}/test_push_pull.py | 0 test_vo_ngas.ini | 18 + test_vo_posix.ini | 17 + 39 files changed, 3039 insertions(+), 7 deletions(-) create mode 100644 pyvospace/server/spaces/ngas/.idea/encodings.xml create mode 100644 pyvospace/server/spaces/ngas/.idea/libraries/R_User_Library.xml create mode 100644 pyvospace/server/spaces/ngas/.idea/misc.xml create mode 100644 pyvospace/server/spaces/ngas/.idea/modules.xml create mode 100644 pyvospace/server/spaces/ngas/.idea/ngas.iml create mode 100644 pyvospace/server/spaces/ngas/.idea/vcs.xml create mode 100644 pyvospace/server/spaces/ngas/.idea/workspace.xml create mode 100644 pyvospace/server/spaces/ngas/__init__.py create mode 100644 pyvospace/server/spaces/ngas/auth.py create mode 100644 pyvospace/server/spaces/ngas/space/__init__.py create mode 100644 pyvospace/server/spaces/ngas/space/__main__.py create mode 100644 pyvospace/server/spaces/ngas/space/ngas_space.py create mode 100644 pyvospace/server/spaces/ngas/storage/__init__.py create mode 100644 pyvospace/server/spaces/ngas/storage/__main__.py create mode 100644 pyvospace/server/spaces/ngas/storage/ngas_storage.py create mode 100644 pyvospace/server/spaces/ngas/utils.py create mode 100755 scripts/start_container_db.sh create mode 100755 scripts/start_ngas.sh create mode 100755 scripts/start_ngas_space.sh create mode 100755 scripts/start_ngas_storage.sh create mode 100644 test/.DS_Store create mode 100644 test/test_ngas/__init__.py create mode 100644 test/test_ngas/test_abort.py create mode 100644 test/test_ngas/test_base.py rename test/{ => test_ngas}/test_copy_move.py (100%) rename test/{ => test_ngas}/test_create.py (100%) create mode 100644 test/test_ngas/test_push_pull.py create mode 100644 test/test_ngas/test_simple.py create mode 100644 test/test_ngas/test_vo_ngas.ini create mode 100644 test/test_posix/__init__.py rename test/{ => test_posix}/test_abort.py (100%) rename test/{ => test_posix}/test_base.py (99%) create mode 100644 test/test_posix/test_copy_move.py create mode 100644 test/test_posix/test_create.py rename test/{ => test_posix}/test_push_pull.py (100%) create mode 100644 test_vo_ngas.ini create mode 100644 test_vo_posix.ini diff --git a/fabfile/APPspecific.py b/fabfile/APPspecific.py index e1afeef..63fa4e5 100644 --- a/fabfile/APPspecific.py +++ b/fabfile/APPspecific.py @@ -164,9 +164,9 @@ def start_APP_and_check_status(): with cd('{0}'.format(APP_source_dir())): virtualenv('python -m unittest discover test') # run('mkdir -p /tmp/fuse') -# virtualenv('posix_space --cfg test_vo.ini > /tmp/space.log 2>&1') +# virtualenv('posix_space --cfg test_vo_posix.ini > /tmp/space.log 2>&1') # time.sleep(2) -# virtualenv('posix_storage --cfg test_vo.ini > /tmp/storage.log 2>&1') +# virtualenv('posix_storage --cfg test_vo_posix.ini > /tmp/storage.log 2>&1') # time.sleep(2) # virtualenv('python -m pyvospace.client.fuse --host localhost --port 8080 --username test --password test --mountpoint /tmp/fuse/`` > /tmp/fusemnt.log 2>&1') # time.sleep(2) @@ -186,9 +186,9 @@ def sysinitstart_APP_and_check_status(): with cd('{0}'.format(APP_source_dir())): # virtualenv('python -m unittest discover test') # run('mkdir -p /tmp/fuse') - # virtualenv('posix_space --cfg test_vo.ini > /tmp/space.log 2>&1') + # virtualenv('posix_space --cfg test_vo_posix.ini > /tmp/space.log 2>&1') # time.sleep(2) - # virtualenv('posix_storage --cfg test_vo.ini > /tmp/storage.log 2>&1') + # virtualenv('posix_storage --cfg test_vo_posix.ini > /tmp/storage.log 2>&1') # time.sleep(2) # virtualenv('python -m pyvospace.client.fuse --host localhost --port 8080 --username test --password test --mountpoint /tmp/fuse/`` > /tmp/fusemnt.log 2>&1') # time.sleep(2) diff --git a/pyvospace/server/spaces/ngas/.idea/encodings.xml b/pyvospace/server/spaces/ngas/.idea/encodings.xml new file mode 100644 index 0000000..15a15b2 --- /dev/null +++ b/pyvospace/server/spaces/ngas/.idea/encodings.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/pyvospace/server/spaces/ngas/.idea/libraries/R_User_Library.xml b/pyvospace/server/spaces/ngas/.idea/libraries/R_User_Library.xml new file mode 100644 index 0000000..71f5ff7 --- /dev/null +++ b/pyvospace/server/spaces/ngas/.idea/libraries/R_User_Library.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/pyvospace/server/spaces/ngas/.idea/misc.xml b/pyvospace/server/spaces/ngas/.idea/misc.xml new file mode 100644 index 0000000..231811a --- /dev/null +++ b/pyvospace/server/spaces/ngas/.idea/misc.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/pyvospace/server/spaces/ngas/.idea/modules.xml b/pyvospace/server/spaces/ngas/.idea/modules.xml new file mode 100644 index 0000000..c4b0829 --- /dev/null +++ b/pyvospace/server/spaces/ngas/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/pyvospace/server/spaces/ngas/.idea/ngas.iml b/pyvospace/server/spaces/ngas/.idea/ngas.iml new file mode 100644 index 0000000..5138a24 --- /dev/null +++ b/pyvospace/server/spaces/ngas/.idea/ngas.iml @@ -0,0 +1,14 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/pyvospace/server/spaces/ngas/.idea/vcs.xml b/pyvospace/server/spaces/ngas/.idea/vcs.xml new file mode 100644 index 0000000..4fce1d8 --- /dev/null +++ b/pyvospace/server/spaces/ngas/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/pyvospace/server/spaces/ngas/.idea/workspace.xml b/pyvospace/server/spaces/ngas/.idea/workspace.xml new file mode 100644 index 0000000..eea2fc6 --- /dev/null +++ b/pyvospace/server/spaces/ngas/.idea/workspace.xml @@ -0,0 +1,106 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - 1548226465905 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file From 679af946ac816a26e70d192765ab206808575a68 Mon Sep 17 00:00:00 2001 From: drtpotter Date: Fri, 1 Mar 2019 16:25:45 +0800 Subject: [PATCH 03/19] Modified create and delete functions in ngas_space --- .idea/encodings.xml | 4 + .idea/misc.xml | 4 + .idea/modules.xml | 8 + .idea/pyvospace.iml | 11 + .idea/vcs.xml | 6 + .idea/workspace.xml | 233 ++++++++++++++++++ .../server/spaces/ngas/space/ngas_space.py | 62 ++--- .../spaces/ngas/storage/ngas_storage.py | 11 +- 8 files changed, 307 insertions(+), 32 deletions(-) create mode 100644 .idea/encodings.xml create mode 100644 .idea/misc.xml create mode 100644 .idea/modules.xml create mode 100644 .idea/pyvospace.iml create mode 100644 .idea/vcs.xml create mode 100644 .idea/workspace.xml diff --git a/.idea/encodings.xml b/.idea/encodings.xml new file mode 100644 index 0000000..15a15b2 --- /dev/null +++ b/.idea/encodings.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..7ba73c2 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..99eb7d6 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/pyvospace.iml b/.idea/pyvospace.iml new file mode 100644 index 0000000..6711606 --- /dev/null +++ b/.idea/pyvospace.iml @@ -0,0 +1,11 @@ + + + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.idea/workspace.xml b/.idea/workspace.xml new file mode 100644 index 0000000..53b1ccc --- /dev/null +++ b/.idea/workspace.xml @@ -0,0 +1,233 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + copy_storage_node + root_dir + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - @@ -152,16 +166,12 @@ - + - - + - - - - + @@ -169,17 +179,18 @@ + + + + + + - - - - - @@ -215,17 +226,37 @@ - + - - + + + + + - - + + + + + + + + + + + + + + + + + + + diff --git a/pyvospace/server/spaces/ngas/auth.py b/pyvospace/server/spaces/ngas/auth.py index 61fca9a..9968aec 100644 --- a/pyvospace/server/spaces/ngas/auth.py +++ b/pyvospace/server/spaces/ngas/auth.py @@ -30,7 +30,6 @@ from pyvospace.core.model import PushToSpace, Property from .utils import statvfs, lstat - PROTECTED_URI = [#'ivo://ivoa.net/vospace/core#title', 'ivo://ivoa.net/vospace/core#creator', #'ivo://ivoa.net/vospace/core#subject', diff --git a/pyvospace/server/spaces/ngas/storage/ngas_storage.py b/pyvospace/server/spaces/ngas/storage/ngas_storage.py index 1113797..6a8029e 100644 --- a/pyvospace/server/spaces/ngas/storage/ngas_storage.py +++ b/pyvospace/server/spaces/ngas/storage/ngas_storage.py @@ -223,7 +223,7 @@ async def upload(self, job: StorageUWSJob, request: aiohttp.web.Request): # Get the UUID for the node id = job.transfer.target.id - ngas_filename=base_name+"_"+str(id) + ngas_filename=f"{base_name}_{id}" if content_length is not None: # Content length exists, we can forward the stream straight to the NGAS server @@ -256,8 +256,7 @@ async def upload(self, job: StorageUWSJob, request: aiohttp.web.Request): # Now the file is on disk, send it nbytes_transfer = await send_file_to_ngas(self.ngas_session, self.ngas_hostname, self.ngas_port, - ngas_filename, stage_file_name, - self.logger) + ngas_filename, stage_file_name) # Remove the staged file if it exists with suppress(Exception): diff --git a/pyvospace/server/spaces/ngas/utils.py b/pyvospace/server/spaces/ngas/utils.py index 76941a7..67c04df 100644 --- a/pyvospace/server/spaces/ngas/utils.py +++ b/pyvospace/server/spaces/ngas/utils.py @@ -206,7 +206,7 @@ async def send_file(request, file_name, file_path): await asyncio.shield(response.write_eof()) -async def send_file_to_ngas(session, hostname, port, filename_ngas, filename_local, logger): +async def send_file_to_ngas(session, hostname, port, filename_ngas, filename_local): """Send a single file to an NGAS server""" try: diff --git a/scripts/.gitignore b/scripts/.gitignore new file mode 100644 index 0000000..7515347 --- /dev/null +++ b/scripts/.gitignore @@ -0,0 +1,2 @@ +./start_container_db.sh +./start_ngas.sh diff --git a/test/test_ngas/test_vo_ngas.ini b/test/test_ngas/test_vo_ngas.ini deleted file mode 100644 index 54210c6..0000000 --- a/test/test_ngas/test_vo_ngas.ini +++ /dev/null @@ -1,21 +0,0 @@ -[Space] -host = localhost -port = 8082 -name = ngas -uri = icrar.org -dsn = postgres://vos_user:vos_user@localhost:5435/vospace -parameters = {} -secret_key = ZlmNyXdQgRhhrC2Wwy-gLZj7Wv6ZtoKH -domain = -use_ssl = 0 - -[Storage] -name = ngas -host = localhost -port = 8083 -parameters = {"root_dir": "/tmp/ngas/storage/", "staging_dir": "/tmp/ngas/staging/"} -use_ssl = 0 -# Use newline-separated values -ngas_servers = 'http://localhost:7777' - 'http://localhost:7777' - From 2074b0e6d951c9cf37af29dc8c325c99b99b237d Mon Sep 17 00:00:00 2001 From: drtpotter Date: Thu, 14 Mar 2019 21:10:53 +0800 Subject: [PATCH 05/19] added container support on upload --- .idea/workspace.xml | 254 +++++++++++++--- .../server/spaces/ngas/space/ngas_space.py | 5 +- .../spaces/ngas/storage/ngas_storage.py | 270 +++++++++++++----- temptest/test_yield.py | 11 + test/test_ngas/test_simple.py | 158 ++++++---- test/test_ngas/test_vo_ngas.ini | 19 ++ 6 files changed, 544 insertions(+), 173 deletions(-) create mode 100644 temptest/test_yield.py create mode 100644 test/test_ngas/test_vo_ngas.ini diff --git a/.idea/workspace.xml b/.idea/workspace.xml index 44d8577..2d89d4b 100644 --- a/.idea/workspace.xml +++ b/.idea/workspace.xml @@ -2,12 +2,10 @@ - - + - - + + + + copy_storage_node - root_dir process_ex send_file_to_ngas + path_to_node_tree + SessionIdentityPolicy + root_dir + stagin_dir + delete + nodes + walk + id + Node + size + uuid + send_stre + filename_ngas + staging_dir @@ -75,18 +136,20 @@ - @@ -96,7 +159,6 @@ - @@ -109,6 +171,12 @@ + + + + + + @@ -130,10 +198,71 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - 1551424720472 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - file://$PROJECT_DIR$/test/test_ngas/test_push_pull.py - 21 - - - file://$PROJECT_DIR$/test/test_ngas/test_abort.py - 21 - - - file://$PROJECT_DIR$/test/test_ngas/test_base.py - 21 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/test/test_ngas/test_vo_ngas.ini b/test/test_ngas/test_vo_ngas.ini index 97de615..c01ae83 100644 --- a/test/test_ngas/test_vo_ngas.ini +++ b/test/test_ngas/test_vo_ngas.ini @@ -13,6 +13,7 @@ use_ssl = 0 name = ngas host = localhost port = 8083 -parameters = {"root_dir": "/tmp/posix/storage/", "staging_dir": "/tmp/posix/staging/"} +parameters = {"root_dir": "/tmp/ngas/storage/", "staging_dir": "/tmp/ngas/staging/"} use_ssl = 0 -ngas_servers=[{"hostname" : "localhost", "port" : 7777}, {"hostname" : "localhost", "port" : 7777}] +ngas_servers=[{"hostname" : "localhost", "port" : 7777}, + {"hostname" : "localhost", "port" : 7777}] diff --git a/test_vo_ngas.ini b/test_vo_ngas.ini index 5860e49..c01ae83 100644 --- a/test_vo_ngas.ini +++ b/test_vo_ngas.ini @@ -13,7 +13,7 @@ use_ssl = 0 name = ngas host = localhost port = 8083 -parameters = {"root_dir": "/tmp/posix/storage/", "staging_dir": "/tmp/posix/staging/"} +parameters = {"root_dir": "/tmp/ngas/storage/", "staging_dir": "/tmp/ngas/staging/"} use_ssl = 0 ngas_servers=[{"hostname" : "localhost", "port" : 7777}, {"hostname" : "localhost", "port" : 7777}] From 791fcbd6e64fc14f20b4a706824d2b90065a6d9a Mon Sep 17 00:00:00 2001 From: drtpotter Date: Fri, 14 Jun 2019 15:50:51 +0800 Subject: [PATCH 16/19] Added documentation and removed root_dir from configuration and runtime files. --- .../server/spaces/ngas/.ngas_plugin.md.swp | Bin 0 -> 12288 bytes pyvospace/server/spaces/ngas/auth.py | 3 +- pyvospace/server/spaces/ngas/ngas_plugin.md | 62 ++++++++++++++++++ .../server/spaces/ngas/space/ngas_space.py | 7 +- .../spaces/ngas/storage/ngas_storage.py | 8 --- test/test_ngas/test_vo_ngas.ini | 2 +- test_vo_ngas.ini | 2 +- 7 files changed, 66 insertions(+), 18 deletions(-) create mode 100644 pyvospace/server/spaces/ngas/.ngas_plugin.md.swp create mode 100644 pyvospace/server/spaces/ngas/ngas_plugin.md diff --git a/pyvospace/server/spaces/ngas/.ngas_plugin.md.swp b/pyvospace/server/spaces/ngas/.ngas_plugin.md.swp new file mode 100644 index 0000000000000000000000000000000000000000..e61305ddb32f5ff518f60e327797dab57756f065 GIT binary patch literal 12288 zcmeHNO>Y}T7@ig=AEhk^)0gvv==3`V?bEHoErm?GBx%D$?S1%h(&%V0~7^ zyd`ygR>{jEFuFc-Y;J0DNUX}H%yDD|Y12OPU3$-d#-CfphWwAVq^H!ABlJK^! z{Q}Rx-OWI5nj_;o*x11XuGRNeo~FkhJ$H9=;&l=$M65W|Nn39V(fF^GvJS%jNJr&0)7O(1>Oh#-oe;);0NFu@C9%H7zOqKyMgZ? zX6!rQ8{iu7Dey7y0dN(V0geLOfvvz_+Zp>4_#OBexDI>;ya!wbUId;5b_0(CyB=a} zC-BRIjNJgf1U>>j1l|GO2A%-+0Na53fZrZq>{s9h@H(&x8VhP5cwhNpp`pe=$Gikj4wIdyn?t8W--zyKT! z*t6EO^M_E=GJt+Ok@d1MgI?&|L^7>0cInb3ra1Nt5|3otn5@W235sRF4M?@g;bPssxIpT!R%=<;k-@dj z6K8)@^;Djwwo-6l?Zj=V4WZDPRECb=043Gxx<~7OpW6_rja0lPXk;X4ww!0|k#v4* zQ5x;*t+M+i|8osY&y@k#y3D)9RxIoXE^v8$?udshuzr8r8VpVN74q71bd z$WK$kyH+DQ6G7CBz)@Q$G3*!tg^*jp3k3Nu;x5?L-85(`nL-hTjsp~_Bh^ZwbQ_RH z<-vYas-#d6O&J2QCc2gKP87#BqyR)ipI}-cUnm22Q$wP6oW(v7J(f!%N`VrND-~NS ztGWtJP^DkJ$}m-I_3Tw!z_ledNZ91!|GsgUp;?!;j%kucS;iH78KE>jUa6o>vB9;q z97d`F_jQwG#rD_oi+W{^IT20L_~`f;bHbWj7a6mW!;+1SEtd6O!e6@sF2M_jswbHi zff8nUMRak;#W*?BeI-2GnrpY!v4a;n-I3O*i;hW>ma7PNMuFn0CRJ-g% zad~f)25}zR{?@`6d(JZ_uuujoLQ%*KZ*a`I-q;-`^pL$^r;$~nD}dVuO|#weFBcuU zh}$tv5V;+ilwk}k4b4qY9X~yd9nbq+j5BtSmx0*eM(D)3`Dw?4?rI|vN@SSZVLh}S zrF}YNT^JalsUQ#_Ggp4ww7uXs%@zuqKYOYS&D}EFHEo%W z)ehT`AwZZ@j-fg6S{^B3d2|DA!9M?redG42u^NT;Nj5-xlpvvX+TF|oVOX>=Z79Z; zuEB#9S9;7+Oh~K3(mF(!+Wqa8T{m+!#NHQ$9mEoRA?^ews3}6B%12G{l-4Aho4g}6 dSAx=&WvDNlVcL`en!862+e`iZ^&@U5`~xwsMg;%> literal 0 HcmV?d00001 diff --git a/pyvospace/server/spaces/ngas/auth.py b/pyvospace/server/spaces/ngas/auth.py index 5ab7a16..b0a3dc8 100644 --- a/pyvospace/server/spaces/ngas/auth.py +++ b/pyvospace/server/spaces/ngas/auth.py @@ -63,11 +63,10 @@ class DBUserNodeAuthorizationPolicy(AbstractAuthorizationPolicy): - def __init__(self, space_name, db_pool, root_dir, ngas_hostname, ngas_port, ngas_session): + def __init__(self, space_name, db_pool, ngas_hostname, ngas_port, ngas_session): super().__init__() self.space_name = space_name self.db_pool = db_pool - self.root_dir = root_dir self.ngas_hostname=ngas_hostname self.ngas_port=ngas_port self.ngas_session=ngas_session diff --git a/pyvospace/server/spaces/ngas/ngas_plugin.md b/pyvospace/server/spaces/ngas/ngas_plugin.md new file mode 100644 index 0000000..e4f16b5 --- /dev/null +++ b/pyvospace/server/spaces/ngas/ngas_plugin.md @@ -0,0 +1,62 @@ +# pyvospace-ngas +NGAS plugin for PyVOSpace, implementation in Python3. + +This plugin provides a PyVOSpace implementation that uses an NGAS server as the backing store. The plugin connects to both a PostgreSQL database and one or more federated instances of an NGAS server. + +**Requirements** + +- Python3.6 or greater. +- Docker support for running a PostgreSQL server. +- Access to a running NGAS server or servers. To setup an NGAS server please see the [online documentation](https://ngas.readthedocs.io/en/latest/). +- FUSE filesystem modules (including fusepy). + +**Installation** + +For installation, including setting up the docker database instance, please see the main ICRAR PyVOSpace [documentation page](README.md). + +**Basic configuration** + +The file *test_vo_ngas.ini* contains a default configuration, as shown below. + +``` +[Space] +host = localhost +port = 8082 +name = ngas +uri = icrar.org +dsn = postgres://vos_user:vos_user@localhost:5435/vospace +parameters = {} +secret_key = ZlmNyXdQgRhhrC2Wwy-gLZj7Wv6ZtoKH +domain = +use_ssl = 0 + +[Storage] +name = ngas +host = localhost +port = 8083 +parameters = {"staging_dir": "/tmp/ngas/staging/"} +use_ssl = 0 +ngas_servers=[{"hostname" : "localhost", "port" : 7777}, + {"hostname" : "localhost", "port" : 7777}] +``` + +Note the *ngas_servers* line. Multiple NGAS servers may be specified here, and it is assumed that the NGAS servers are synchronised and can work with the same information. At runtime an NGAS server will be chosen at random for both the space and storage servers. + +**Running the Space and Storage servers** + +'ngas_space --cfg test_vo_ngas.ini' +'ngas_storage --cfg test_vo_ngas.ini' + +**Unit tests** + +Unit tests are located in test/test_ngas/\*.py and assume an NGAS server is running on localhost at port 7777. + +**Common errors** + +If you see an error such as, + +``` +raise aiohttp.web.HTTPServerError(reason=f"File {filename_ngas} not deleted properly from NGAS server"), +``` + +it means that the NGAS server is not configured with caching enabled and has refused to delete a file. In this instance check if you want the NGAS server to be able to delete files and then enable caching in the NGAS server configuration. diff --git a/pyvospace/server/spaces/ngas/space/ngas_space.py b/pyvospace/server/spaces/ngas/space/ngas_space.py index c809860..6abfb8d 100644 --- a/pyvospace/server/spaces/ngas/space/ngas_space.py +++ b/pyvospace/server/spaces/ngas/space/ngas_space.py @@ -82,10 +82,6 @@ def __init__(self, cfg_file, *args, **kwargs): self.ngas_port=int(self.ngas_server["port"]) self.ngas_session = aiohttp.ClientSession() - self.root_dir = self.storage_parameters['root_dir'] - #if not self.root_dir: - # raise Exception('root_dir not found.') - self.staging_dir = self.storage_parameters['staging_dir'] if not self.staging_dir: raise Exception('staging_dir not found.') @@ -110,7 +106,6 @@ async def setup_space(self): SessionIdentityPolicy(), DBUserNodeAuthorizationPolicy(self['space_name'], self['db_pool'], - self.root_dir, self.ngas_hostname, self.ngas_port, self.ngas_session)) @@ -250,4 +245,4 @@ async def get_transfer_protocols(self, job: UWSJob) -> List[Protocol]: if not new_protocols: raise VOSpaceError(400, "Protocol Not Supported. No storage found") - return new_protocols \ No newline at end of file + return new_protocols diff --git a/pyvospace/server/spaces/ngas/storage/ngas_storage.py b/pyvospace/server/spaces/ngas/storage/ngas_storage.py index fb82124..9f06429 100644 --- a/pyvospace/server/spaces/ngas/storage/ngas_storage.py +++ b/pyvospace/server/spaces/ngas/storage/ngas_storage.py @@ -69,9 +69,6 @@ def __init__(self, cfg_file, *args, **kwargs): self.ngas_port=int(self.ngas_server["port"]) self.ngas_session = aiohttp.ClientSession() - # Do we need a root_dir, probably not. - self.root_dir = self.parameters['root_dir'] - # We need a staging directory self.staging_dir = self.parameters['staging_dir'] if not self.staging_dir: @@ -91,9 +88,6 @@ async def shutdown(self): async def setup(self): await super().setup() - # await mkdir(self.root_dir) - # Probably don't need to make the root directory - # We do need a staging directory await mkdir(self.staging_dir) @@ -108,7 +102,6 @@ async def setup(self): SessionIdentityPolicy(), DBUserNodeAuthorizationPolicy( self.name, self.db_pool, - self.root_dir, self.ngas_hostname, self.ngas_port, self.ngas_session)) @@ -121,7 +114,6 @@ async def create(cls, cfg_file, *args, **kwargs): async def download(self, job: StorageUWSJob, request: aiohttp.web.Request): """Download files from the NGAS server""" - root_dir = self.root_dir path_tree = job.transfer.target.path if job.transfer.target.node_type == NodeType.ContainerNode: diff --git a/test/test_ngas/test_vo_ngas.ini b/test/test_ngas/test_vo_ngas.ini index c01ae83..7e9e1e1 100644 --- a/test/test_ngas/test_vo_ngas.ini +++ b/test/test_ngas/test_vo_ngas.ini @@ -13,7 +13,7 @@ use_ssl = 0 name = ngas host = localhost port = 8083 -parameters = {"root_dir": "/tmp/ngas/storage/", "staging_dir": "/tmp/ngas/staging/"} +parameters = {"staging_dir": "/tmp/ngas/staging/"} use_ssl = 0 ngas_servers=[{"hostname" : "localhost", "port" : 7777}, {"hostname" : "localhost", "port" : 7777}] diff --git a/test_vo_ngas.ini b/test_vo_ngas.ini index c01ae83..7e9e1e1 100644 --- a/test_vo_ngas.ini +++ b/test_vo_ngas.ini @@ -13,7 +13,7 @@ use_ssl = 0 name = ngas host = localhost port = 8083 -parameters = {"root_dir": "/tmp/ngas/storage/", "staging_dir": "/tmp/ngas/staging/"} +parameters = {"staging_dir": "/tmp/ngas/staging/"} use_ssl = 0 ngas_servers=[{"hostname" : "localhost", "port" : 7777}, {"hostname" : "localhost", "port" : 7777}] From a85e3c2df1cc88475a19aee252435bd78182aa83 Mon Sep 17 00:00:00 2001 From: drtpotter Date: Fri, 14 Jun 2019 15:52:25 +0800 Subject: [PATCH 17/19] Added documentation and removed root_dir from configuration --- .../server/spaces/ngas/.ngas_plugin.md.swp | Bin 12288 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 pyvospace/server/spaces/ngas/.ngas_plugin.md.swp diff --git a/pyvospace/server/spaces/ngas/.ngas_plugin.md.swp b/pyvospace/server/spaces/ngas/.ngas_plugin.md.swp deleted file mode 100644 index e61305ddb32f5ff518f60e327797dab57756f065..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeHNO>Y}T7@ig=AEhk^)0gvv==3`V?bEHoErm?GBx%D$?S1%h(&%V0~7^ zyd`ygR>{jEFuFc-Y;J0DNUX}H%yDD|Y12OPU3$-d#-CfphWwAVq^H!ABlJK^! z{Q}Rx-OWI5nj_;o*x11XuGRNeo~FkhJ$H9=;&l=$M65W|Nn39V(fF^GvJS%jNJr&0)7O(1>Oh#-oe;);0NFu@C9%H7zOqKyMgZ? zX6!rQ8{iu7Dey7y0dN(V0geLOfvvz_+Zp>4_#OBexDI>;ya!wbUId;5b_0(CyB=a} zC-BRIjNJgf1U>>j1l|GO2A%-+0Na53fZrZq>{s9h@H(&x8VhP5cwhNpp`pe=$Gikj4wIdyn?t8W--zyKT! z*t6EO^M_E=GJt+Ok@d1MgI?&|L^7>0cInb3ra1Nt5|3otn5@W235sRF4M?@g;bPssxIpT!R%=<;k-@dj z6K8)@^;Djwwo-6l?Zj=V4WZDPRECb=043Gxx<~7OpW6_rja0lPXk;X4ww!0|k#v4* zQ5x;*t+M+i|8osY&y@k#y3D)9RxIoXE^v8$?udshuzr8r8VpVN74q71bd z$WK$kyH+DQ6G7CBz)@Q$G3*!tg^*jp3k3Nu;x5?L-85(`nL-hTjsp~_Bh^ZwbQ_RH z<-vYas-#d6O&J2QCc2gKP87#BqyR)ipI}-cUnm22Q$wP6oW(v7J(f!%N`VrND-~NS ztGWtJP^DkJ$}m-I_3Tw!z_ledNZ91!|GsgUp;?!;j%kucS;iH78KE>jUa6o>vB9;q z97d`F_jQwG#rD_oi+W{^IT20L_~`f;bHbWj7a6mW!;+1SEtd6O!e6@sF2M_jswbHi zff8nUMRak;#W*?BeI-2GnrpY!v4a;n-I3O*i;hW>ma7PNMuFn0CRJ-g% zad~f)25}zR{?@`6d(JZ_uuujoLQ%*KZ*a`I-q;-`^pL$^r;$~nD}dVuO|#weFBcuU zh}$tv5V;+ilwk}k4b4qY9X~yd9nbq+j5BtSmx0*eM(D)3`Dw?4?rI|vN@SSZVLh}S zrF}YNT^JalsUQ#_Ggp4ww7uXs%@zuqKYOYS&D}EFHEo%W z)ehT`AwZZ@j-fg6S{^B3d2|DA!9M?redG42u^NT;Nj5-xlpvvX+TF|oVOX>=Z79Z; zuEB#9S9;7+Oh~K3(mF(!+Wqa8T{m+!#NHQ$9mEoRA?^ews3}6B%12G{l-4Aho4g}6 dSAx=&WvDNlVcL`en!862+e`iZ^&@U5`~xwsMg;%> From 6cc23e0efe2dc87151d7a3e0026f52a5fdfb8b90 Mon Sep 17 00:00:00 2001 From: drtpotter Date: Fri, 14 Jun 2019 15:54:41 +0800 Subject: [PATCH 18/19] Removed .idea directory --- .idea/libraries/R_User_Library.xml | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 .idea/libraries/R_User_Library.xml diff --git a/.idea/libraries/R_User_Library.xml b/.idea/libraries/R_User_Library.xml deleted file mode 100644 index 71f5ff7..0000000 --- a/.idea/libraries/R_User_Library.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file From 107e6d2b282089626cd1faba48c2269012adfa2c Mon Sep 17 00:00:00 2001 From: drtpotter Date: Fri, 14 Jun 2019 15:58:10 +0800 Subject: [PATCH 19/19] Corrected documentation file. --- pyvospace/server/spaces/ngas/{ngas_plugin.md => README.md} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename pyvospace/server/spaces/ngas/{ngas_plugin.md => README.md} (94%) diff --git a/pyvospace/server/spaces/ngas/ngas_plugin.md b/pyvospace/server/spaces/ngas/README.md similarity index 94% rename from pyvospace/server/spaces/ngas/ngas_plugin.md rename to pyvospace/server/spaces/ngas/README.md index e4f16b5..f25875c 100644 --- a/pyvospace/server/spaces/ngas/ngas_plugin.md +++ b/pyvospace/server/spaces/ngas/README.md @@ -12,7 +12,7 @@ This plugin provides a PyVOSpace implementation that uses an NGAS server as the **Installation** -For installation, including setting up the docker database instance, please see the main ICRAR PyVOSpace [documentation page](README.md). +For installation, including setting up the docker database instance, please see the main ICRAR PyVOSpace [documentation page](https://github.com/ICRAR/pyvospace/blob/master/README.md). **Basic configuration**