dataset updated

Example/pizza.yaml

-id: http://example.org/pizza-schema
-name: PizzaSchema
+id: http://example.org/pizza
+name: pizza
 description: A schema for validating pizza data
 prefixes:
   linkml: https://w3id.org/linkml/
@@ -20,101 +20,126 @@ enums:
         description: Medium level of spiciness
       Mild:
         description: Mild level of spiciness
-
-classes:
-  Country:
-    attributes:
-      id:
-        identifier: true
-        range: string
-      title:
-        range: string
-
+  
   PizzaBase:
-    attributes:
-      id:
-        identifier: true
-        range: string
-      title:
-        range: string
-
-  PizzaTopping:
-    attributes:
-      id:
-        identifier: true
-        range: string
-      title:
-        range: string
-      Spiciness:
-        inlined_as_list: true
-        range: SpicinessLevel
-        multivalued: true
-        required: false
+    permissible_values:
+      DeepPanBase:
+        meaning: piz:DeepPanBase
+        description: Thick Base
+      ThinAndCrispyBase:
+        meaning: piz:ThinAndCrispyBase
+        description: Thin Base
 
+classes:
   Pizza:
     attributes:
-      id:
-        identifier: true
-        range: string
-        required: true
       title:
         range: string
-        required: true
-      description:
-        range: string
       Base:
-        inlined_as_list: true
         range: PizzaBase
-        multivalued: true
       Topping:
-        inlined_as_list: true
-        range: PizzaTopping
+        range: string
         multivalued: true
       CountryOfOrigin:
-        inlined_as_list: true
-        range: Country
-        multivalued: true
+        range: string
+      Spiciness:
+        range: SpicinessLevel
 
-  PizzaCollection:
-    attributes:
-      pizzas:
-        inlined_as_list: true
-        range: Pizza
-        multivalued: true
 
-  CountryCollection:
-    attributes:
-      countries:
-        range: Country
-        multivalued: true
+# classes:
+#   Country:
+#     attributes:
+#       id:
+#         identifier: true
+#         range: string
+#       title:
+#         range: string
 
-  PizzaBaseCollection:
-    attributes:
-      pizza_bases:
-        range: PizzaBase
-        multivalued: true
+#   PizzaBase:
+#     attributes:
+#       id:
+#         identifier: true
+#         range: string
+#       title:
+#         range: string
 
-  PizzaToppingCollection:
-    attributes:
-      pizza_toppings:
-        range: PizzaTopping
-        multivalued: true
+#   PizzaTopping:
+#     attributes:
+#       id:
+#         identifier: true
+#         range: string
+#       title:
+#         range: string
+#       Spiciness:
+#         inlined_as_list: true
+#         range: SpicinessLevel
+#         multivalued: true
+#         required: false
 
-  SpicinessLevelCollection:
-    attributes:
-      spiciness_levels:
-        range: SpicinessLevel
-        multivalued: true
+#   Pizza:
+#     attributes:
+#       id:
+#         identifier: true
+#         range: string
+#         required: true
+#       title:
+#         range: string
+#         required: true
+#       description:
+#         range: string
+#       Base:
+#         inlined_as_list: true
+#         range: PizzaBase
+#         multivalued: true
+#       Topping:
+#         inlined_as_list: true
+#         range: PizzaTopping
+#         multivalued: true
+#       CountryOfOrigin:
+#         inlined_as_list: true
+#         range: Country
+#         multivalued: true
 
-  Root:
-    attributes:
-      pizzas:
-        range: PizzaCollection
-      countries:
-        range: CountryCollection
-      pizza_bases:
-        range: PizzaBaseCollection
-      pizza_toppings:
-        range: PizzaToppingCollection
-      spiciness_levels:
-        range: SpicinessLevelCollection
+#   PizzaCollection:
+#     attributes:
+#       pizzas:
+#         inlined_as_list: true
+#         range: Pizza
+#         multivalued: true
+
+#   CountryCollection:
+#     attributes:
+#       countries:
+#         range: Country
+#         multivalued: true
+
+#   PizzaBaseCollection:
+#     attributes:
+#       pizza_bases:
+#         range: PizzaBase
+#         multivalued: true
+
+#   PizzaToppingCollection:
+#     attributes:
+#       pizza_toppings:
+#         range: PizzaTopping
+#         multivalued: true
+
+#   SpicinessLevelCollection:
+#     attributes:
+#       spiciness_levels:
+#         range: SpicinessLevel
+#         multivalued: true
+
+#   Root:
+#     attributes:
+#       pizzas:
+#         range: PizzaCollection
+#       countries:
+#         range: CountryCollection
+#       pizza_bases:
+#         range: PizzaBaseCollection
+#       pizza_toppings:
+#         range: PizzaToppingCollection
+#       spiciness_levels:
+#         range: SpicinessLevelCollection

Example/pizza_dataset.yaml

-pizzas: 
-    - id: "pizza1"
+
+datasetName: "Margherita"
+description: "Classic Margherita pizza with tomato, mozzarella, and basil."
+principalInvestigator: "Dr. Bunsen Honeydew"
+owner: "Anjali"
+contactEmail: "anjali@example.com"
+sourceFolder: "path"
+size: 123456
+creationLocation: "magrathea"
+creationTime: "2024-06-12T12:00:00Z"
+type: "raw"
+instrumentId: "Instrument001"
+proposalId: "Proposal001"
+dataFormat: "HDF5"
+isPublished: False
+ownerGroup: "hasylab"
+accessGroups: ["Group001"]
+keywords: ["example", "data", "keywords"]
+license: "CC BY 4.0"
+scientificMetadata:
       title: "Margherita"
-      description: "Classic Margherita pizza with tomato, mozzarella, and basil."
-      Base:
-        - id: "base1"
-          title: "Thin crust"
-      Topping:
-        - id: "topping1"
-          title: "Mozzarella"
-          Spiciness: ['Hot',Medium,'Mild']
-        - id: "topping2"
-          title: "Tomato"
-          Spiciness: []
-      CountryOfOrigin:
-        - id: "country1"
-          title: "Italy"
-    - id: "pizza2"
-      title: "Fungi"
-      description: "Classic Fungi pizza with tomato, mozzarella, and Fungi."
-      Base:
-        - id: "base1"
-          title: "Thin crust"
-      Topping:
-        - id: "topping1"
-          title: "Mozzarella"
-          Spiciness: ['Hot',Medium,'Mild']
-        - id: "topping2"
-          title: "Tomato"
-          Spiciness: []
-      CountryOfOrigin:
-        - id: "country1"
-          title: "Italy"
\ No newline at end of file
+      Base: "ThinAndCrispyBase"
+      Topping: ["Mozzarella", "Tomato"]
+      CountryOfOrigin: "Italy"
+  # pizzas:
+  #     id: "pizza1"
+  #     title: "Margherita"
+  #     description: "Classic Margherita pizza with tomato, mozzarella, and basil."
+  #     Base:
+  #       - id: "base1"
+  #         title: "Thin crust"
+  #     Topping:
+  #       - id: "topping1"
+  #         title: "Mozzarella"
+  #         Spiciness: ['Hot']
+  #       # - id: "topping2"
+  #       #   title: "Tomato"
+  #       #   Spiciness: []
+  #     CountryOfOrigin:
+  #       - id: "country1"
+  #         title: "Italy"
+          
+  #   - id: "pizza2"
+  #     title: "Fungi"
+  #     description: "Classic Fungi pizza with tomato, mozzarella, and Fungi."
+  #     Base:
+  #       - id: "base1"
+  #         title: "Thin crust"
+  #     Topping:
+  #       - id: "topping1"
+  #         title: "Mozzarella"
+  #         Spiciness: ['Hot',Medium,'Mild']
+  #       - id: "topping2"
+  #         title: "Tomato"
+  #         Spiciness: []
+  #     CountryOfOrigin:
+  #       - id: "country1"
+  #         title: "Italy"
\ No newline at end of file

Example/scicatdataset.yaml

+# --
+# # DESY Public Dataset Metadata definition © 2024 by Deutsches
+# # Elektronen-Synchrotron DESY is licensed under CC BY 4.0
+id: https://desy.de/linkml/opendata/pizza2
+name: pizza2
+prefixes:
+  linkml: https://w3id.org/linkml/
+  UO: http://purl.obolibrary.org/obo/UO_
+  qudt: http://qudt.org/schema/qudt/
+  unit: http://qudt.org/vocab/unit/
+  OM: http://www.ontology-of-units-of-measure.org/resource/om-2/
+  nerc: http://vocab.nerc.ac.uk/collection/P06/current/
+#   # nexus: 
+imports:
+  - linkml:types
+  - pizza
+  - ../schema/base_scicat_dataset
+  # - ../schema/base_catalogue_metadata
+  # - pizza
+# default_range: string
+classes:
+  ScientificMetadataCommons:
+    description: some optional common properties
+    attributes:
+        CustomParams:
+            range: object
+            required: false
+
+  PizzaScientificMetadata:
+    description: The metadata associated with Pizza
+    is_a: Pizza
+    mixins: [ScientificMetadataCommons]
+  Pizzadataset:
+    tree_root: true
+    is_a: ScicatDataset
+    description: The metadata associated with P65
+    attributes:
+      scientificMetadata:
+        range: PizzaScientificMetadata
+        required: true
\ No newline at end of file

Example/validate_and_ingest.py

@@ -14,31 +14,10 @@ with open('pizza_dataset.yaml') as yaml_data:
     instance = yaml.safe_load(yaml_data)
 
 
-report = validate(instance, "pizza.yaml", "PizzaCollection")
+report = validate(instance, "scicatdataset.yaml")
 SCICAT_API_URL = "http://scicat-rockit-p65-test1.desy.de/api/v3"
-TOKEN = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJfaWQiOiI2NTAxN2M1ODY4Yjg4NWVlMzJkYWQ4YzgiLCJ1c2VybmFtZSI6ImluZ2VzdG9yIiwiZW1haWwiOiJzY2ljYXRpbmdlc3RvckB5b3VyLnNpdGUiLCJhdXRoU3RyYXRlZ3kiOiJsb2NhbCIsIl9fdiI6MCwiaWQiOiI2NTAxN2M1ODY4Yjg4NWVlMzJkYWQ4YzgiLCJpYXQiOjE3MTg2MTU4NTYsImV4cCI6MTcxODY1OTA1Nn0.Zdcq8Qy93HnVBZBFb5DhnIn0Zq71FUwCmObXJSfCdZo"
-
-dataset_metadata = {
-        "principalInvestigator": "Dr. John Doe",
-        "owner": "Institution Name",
-        "contactEmail": "anjali@example.com",
-        "sourceFolder": "i am new",
-        "scientificMetadata":instance,
-        # "pid": "sdfsdfkjsdhfsd",
-        "size": 123456,
-        "creationLocation": "magrathea",
-        "creationTime": "2024-06-12T12:00:00Z",
-        "type": "raw",
-        "instrumentId": "Instrument001",
-        "proposalId": "Proposal001",
-        "dataFormat": "HDF5",
-        "isPublished": False,
-        "ownerGroup": "hasylab",
-        "accessGroups": ["Group001"],
-        "description": "Description of the dataset",
-        "keywords": ["example", "data", "keywords"],
-        "license": "CC BY 4.0"
-    }
+TOKEN = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJfaWQiOiI2NTAxN2M1ODY4Yjg4NWVlMzJkYWQ4YzgiLCJ1c2VybmFtZSI6ImluZ2VzdG9yIiwiZW1haWwiOiJzY2ljYXRpbmdlc3RvckB5b3VyLnNpdGUiLCJhdXRoU3RyYXRlZ3kiOiJsb2NhbCIsIl9fdiI6MCwiaWQiOiI2NTAxN2M1ODY4Yjg4NWVlMzJkYWQ4YzgiLCJpYXQiOjE3MTkzOTcxMjgsImV4cCI6MTcxOTQ0MDMyOH0.nO0ja0-u1t9Sa-sRhUf4JmUwGLneand62Zq2DdfR8WQ"
+dataset_metadata = instance
 
 client = SciCatClient(SCICAT_API_URL, TOKEN)
     

schema/base_scicat_dataset.yaml

@@ -112,14 +112,18 @@ classes:
       dataFormat:
         description: Defines the format of the data files in this dataset, e.g Nexus
           Version x.y.
-      # proposalId:
-        # description: The ID of the proposal to which the dataset belongs.
+      proposalId:
+        description: The ID of the proposal to which the dataset belongs.
+      isPublished:
+        description: The ID of the proposal to which the dataset belongs.
       #~ sampleId:
         #~ description: ID of the sample used when collecting the data.
       instrumentId:
         description: ID of the instrument where the data was created.
-    #   #~ ownerGroup: {}
-    #   #~ accessGroups: {}
+      ownerGroup: 
+        description: Defines the group which owns the data, and therefore has unrestricted access to this data. Usually a pgroup like p12151
+      accessGroups: 
+        description: Defines the group which owns the data, and therefore has unrestricted access to this data. Usually a pgroup like p12151
     #   #~ instrumentGroup: {}