python interface for datasets

tests/automatic/worker/worker_api/check_linux.sh

@@ -39,7 +39,7 @@ do
 	images=''
 	for j in `seq 1 3`;
 	do
-		images="$images,{"_id":$j,"size":100,"name":'$i_$j',"lastchange":1,"source":'none',"buf_id":0,"meta":{"test":10}}"
+		images="$images,{"_id":$j,"size":100,"name":'${i}_${j}',"lastchange":1,"source":'none',"buf_id":0,"meta":{"test":10}}"
 	done
 	images=${images#?}
 	echo 'db.data.insert({"_id":'$i',"size":3,"images":['$images']})' | mongo ${database_name}

tests/automatic/worker/worker_api_python/check_linux.sh

@@ -30,9 +30,24 @@ sleep 1
 
 export PYTHONPATH=$1:${PYTHONPATH}
 
-python worker_api.py 127.0.0.1:8400 $source_path $database_name $token_test_run
+python worker_api.py 127.0.0.1:8400 $source_path $database_name $token_test_run single
 
 
+#check datasets
+echo "db.dropDatabase()" | mongo ${database_name}
 
+sleep 1
+
+for i in `seq 1 10`;
+do
+	images=''
+	for j in `seq 1 3`;
+	do
+		images="$images,{"_id":$j,"size":100,"name":'${i}_${j}',"lastchange":1,"source":'none',"buf_id":0,"meta":{"test":10}}"
+	done
+	images=${images#?}
+	echo 'db.data.insert({"_id":'$i',"size":3,"images":['$images']})' | mongo ${database_name}
+done
 
 
+python worker_api.py 127.0.0.1:8400 $source_path $database_name $token_test_run dataset

tests/automatic/worker/worker_api_python/worker_api.py

@@ -35,76 +35,115 @@ def assert_eq(val,expected,name):
         print ('val: ', val,' expected: ',expected)
         sys.exit(1)
 
+def check_single(broker,group_id_new):
+    _, meta, err = broker.get_next(group_id_new, meta_only=True)
+    assert_noterr(err, "get_next")
+    assert_metaname(meta,"1","get next1")
+    assert_usermetadata(meta,"get next1")
 
-source, path, beamtime, token = sys.argv[1:]
 
-broker, err = asapo_worker.create_server_broker(source,path, beamtime,token,1000)
+    _, meta, err = broker.get_next(group_id_new, meta_only=True)
+    assert_noterr(err, "get_next2")
+    assert_metaname(meta,"2","get next2")
+    assert_usermetadata(meta,"get next2")
 
-group_id_new, err = broker.generate_group_id()
-assert_noterr(err, "generate_group")
+    _, meta, err = broker.get_last(group_id_new, meta_only=True)
+    assert_noterr(err, "get_last1")
+    assert_metaname(meta,"5","get last1")
+    assert_usermetadata(meta,"get last1")
+
+    _, meta, err = broker.get_next(group_id_new, meta_only=True)
+    assert_err(err, "get_next3")
+
+    size,err = broker.get_ndatasets()
+    assert_noterr(err, "get_ndatasets")
+    assert_eq(size,5,"get_ndatasets")
+
+
+    err = broker.reset_counter(group_id_new)
+    assert_noterr(err, "reset_counter")
+
+    _, meta, err = broker.get_next(group_id_new, meta_only=True)
+    assert_noterr(err, "get_next4")
+    assert_metaname(meta,"1","get next4")
+    assert_usermetadata(meta,"get next4")
 
-_, meta, err = broker.get_next(group_id_new, meta_only=True)
-assert_noterr(err, "get_next")
-assert_metaname(meta,"1","get next1")
-assert_usermetadata(meta,"get next1")
 
+    _, meta, err = broker.get_by_id(3, group_id_new, meta_only=True)
+    assert_noterr(err, "get_by_id")
+    assert_metaname(meta,"3","get get_by_id")
+    assert_usermetadata(meta,"get get_by_id")
 
-_, meta, err = broker.get_next(group_id_new, meta_only=True)
-assert_noterr(err, "get_next2")
-assert_metaname(meta,"2","get next2")
-assert_usermetadata(meta,"get next2")
+    _, meta, err = broker.get_next(group_id_new, meta_only=True)
+    assert_noterr(err, "get_next5")
+    assert_metaname(meta,"4","get next5")
+    assert_usermetadata(meta,"get next5")
 
-_, meta, err = broker.get_last(group_id_new, meta_only=True)
-assert_noterr(err, "get_last1")
-assert_metaname(meta,"5","get last1")
-assert_usermetadata(meta,"get last1")
 
-_, meta, err = broker.get_next(group_id_new, meta_only=True)
-assert_err(err, "get_next3")
+    images,err = broker.query_images("meta.test = 10")
+    assert_noterr(err, "query1")
+    assert_eq(len(images),5,"size of query answer 1")
+    for image in images:
+        assert_usermetadata(image,"query_images")
 
-size,err = broker.get_ndatasets()
-assert_noterr(err, "get_ndatasets")
-assert_eq(size,5,"get_ndatasets")
 
+    images,err = broker.query_images("meta.test = 10 AND name='1'")
+    assert_eq(len(images),1,"size of query answer 2 ")
+    assert_noterr(err, "query2")
 
-err = broker.reset_counter(group_id_new)
-assert_noterr(err, "reset_counter")
+    for image in images:
+        assert_usermetadata(image,"query_images")
 
-_, meta, err = broker.get_next(group_id_new, meta_only=True)
-assert_noterr(err, "get_next4")
-assert_metaname(meta,"1","get next4")
-assert_usermetadata(meta,"get next4")
+    images,err = broker.query_images("meta.test = 11")
+    assert_eq(len(images),0,"size of query answer 3 ")
+    assert_noterr(err, "query3")
 
+    images,err = broker.query_images("bla")
+    assert_err(err, "wrong query")
 
-_, meta, err = broker.get_by_id(3, group_id_new, meta_only=True)
-assert_noterr(err, "get_by_id")
-assert_metaname(meta,"3","get get_by_id")
-assert_usermetadata(meta,"get get_by_id")
+def check_dataset(broker,group_id_new):
+    id, metas, err = broker.get_next_dataset(group_id_new)
+    assert_noterr(err, "get_next_dataset1")
+    assert_eq(id,1,"get_next_dataset1")
+    assert_metaname(metas[0],"1_1","get nextdataset1 name1")
+    assert_metaname(metas[1],"1_2","get nextdataset1 name2")
+    assert_usermetadata(metas[0],"get nextdataset1 meta")
 
-_, meta, err = broker.get_next(group_id_new, meta_only=True)
-assert_noterr(err, "get_next5")
-assert_metaname(meta,"4","get next5")
-assert_usermetadata(meta,"get next5")
+    id, metas, err = broker.get_next_dataset(group_id_new)
+    assert_noterr(err, "get_next_dataset2")
+    assert_eq(id,2,"get_next_dataset2")
+    assert_metaname(metas[0],"2_1","get nextdataset2 name1")
 
+    id, metas, err = broker.get_last_dataset(group_id_new)
+    assert_noterr(err, "get_last_dataset1")
+    assert_eq(id,10,"get_last_dataset1")
+    assert_metaname(metas[2],"10_3","get get_last_dataset1 name3")
 
-images,err = broker.query_images("meta.test = 10")
-assert_noterr(err, "query1")
-assert_eq(len(images),5,"size of query answer 1")
-for image in images:
-    assert_usermetadata(image,"query_images")
+    id, metas, err = broker.get_next_dataset(group_id_new)
+    assert_eq(id,None,"get_next_dataset3 id")
+    assert_eq(id,metas,"get_next_dataset3 metas")
+    assert_err(err, "get_next_dataset3 err")
 
+    id, metas, err = broker.get_dataset_by_id(8,group_id_new)
+    assert_eq(id,8,"get_dataset_by_id1 id")
+    assert_noterr(err, "get_dataset_by_id1 err")
+    assert_metaname(metas[2],"8_3","get get_dataset_by_id1 name3")
 
-images,err = broker.query_images("meta.test = 10 AND name='1'")
-assert_eq(len(images),1,"size of query answer 2 ")
-assert_noterr(err, "query2")
+    id, metas, err = broker.get_next_dataset(group_id_new)
+    assert_eq(id,9,"get_next_dataset4 id")
+
+
+source, path, beamtime, token, mode = sys.argv[1:]
+
+broker, err = asapo_worker.create_server_broker(source,path, beamtime,token,1000)
+
+group_id_new, err = broker.generate_group_id()
+assert_noterr(err, "generate_group")
 
-for image in images:
-    assert_usermetadata(image,"query_images")
 
-images,err = broker.query_images("meta.test = 11")
-assert_eq(len(images),0,"size of query answer 3 ")
-assert_noterr(err, "query3")
+if mode == "single":
+    check_single(broker,group_id_new)
 
-images,err = broker.query_images("bla")
-assert_err(err, "wrong query")
+if mode == "dataset":
+    check_dataset(broker,group_id_new)
 

worker/api/python/asapo_worker.pxd

@@ -25,7 +25,9 @@ cdef extern from "asapo_worker.h" namespace "asapo":
   cppclass FileInfos:
     vector[FileInfo].iterator begin()
     vector[FileInfo].iterator end()
-
+  struct DataSet:
+    uint64_t id
+    FileInfos content
 
 
 cdef extern from "asapo_worker.h" namespace "asapo":
@@ -40,6 +42,9 @@ cdef extern from "asapo_worker.h" namespace "asapo":
         string GenerateNewGroupId(Error* err)
         string GetBeamtimeMeta(Error* err)
         FileInfos QueryImages(string query, Error* err)
+        DataSet GetNextDataset(string group_id, Error* err)
+        DataSet GetLastDataset(string group_id, Error* err)
+        DataSet GetDatasetById(uint64_t id,string group_id, Error* err)
 
 
 cdef extern from "asapo_worker.h" namespace "asapo":

worker/api/python/asapo_worker.pyx.in

@@ -94,6 +94,32 @@ cdef class PyDataBroker:
             for fi in file_infos:
                 json_list.append(json.loads(_str(fi.Json())))
             return json_list, None
+    def _op_dataset(self, op, group_id, id):
+        cdef FileInfos file_infos
+        cdef DataSet dataset
+        cdef Error err
+        if op == "next":
+            dataset = self.c_broker.GetNextDataset(_bytes(group_id),&err)
+        elif op == "last":
+            dataset = self.c_broker.GetLastDataset(_bytes(group_id),&err)
+        elif op == "id":
+            dataset = self.c_broker.GetDatasetById(id,_bytes(group_id),&err)
+        err_str = _str(GetErrorString(&err))
+        if err_str.strip():
+            return None, None, err_str
+        else:
+            json_list = []
+            for fi in dataset.content:
+                json_list.append(json.loads(_str(fi.Json())))
+            return dataset.id, json_list, None
+
+
+    def get_next_dataset(self, group_id):
+        return self._op_dataset("next",group_id,0)
+    def get_last_dataset(self, group_id):
+        return self._op_dataset("last",group_id,0)
+    def get_dataset_by_id(self, id, group_id):
+        return self._op_dataset("id",group_id,id)
 
 
     def get_beamtime_meta(self):