Merge branch '16-xas-catalysis' into 'main'

Resolve "XAS catalysis"

Closes #16

See merge request !16

.gitlab-ci.yml

@@ -21,15 +21,15 @@ lint-all:
 validate_p08:
     stage: validate
     script:
-        - linkml-validate -s ./schema/beamline_p08.yaml ./Dataset/p08_example_data.yaml
-        - linkml-validate -s ./schema/beamline_p65.yaml ./Dataset/p65_example_data.yaml
+        # - linkml-validate -s ./schema/beamline_p08.yaml ./Dataset/p08_example_data.yaml
+        - linkml-validate -s ./schema/beamline_p65_xas.yaml ./Dataset/p65_example_data.yaml
 
 # Stage - build derivative - generate json schema
 json-schema:
   stage: build-derivatives
   script:
-    - gen-json-schema --closed ./schema/beamline_p08.yaml  > beamline_p08.json
-    - gen-json-schema --closed ./schema/beamline_p65.yaml  > beamline_p65.json
+    # - gen-json-schema --closed ./schema/beamline_p08.yaml  > beamline_p08.json
+    - gen-json-schema --closed ./schema/beamline_p65_xas.yaml  > beamline_p65_xas.json
   artifacts:
     name: "json-schemas"
     paths:
@@ -43,22 +43,31 @@ gen_docs:
         - "docs/"
 
 #Stage - build docs
-build_docs:
+build_html:
     image: squidfunk/mkdocs-material 
     stage: build-docs
     needs: [gen_docs]
     before_script:
         - pip install mkdocs-mermaid2-plugin
     script:
-        - mkdocs build -d build
+        - mkdocs build -d build/html
     artifacts:
         paths:
-        - "build/"
+        - "build/html"
 
+build_jsonld:
+    stage: build-docs
+    needs: [gen_docs]
+    script:
+        - python ./scripts/gen_allformat.py -i ./schema/beamline_p65_xas.yaml -o build/beamline_p65_xas.json
+    artifacts:
+        paths:
+        - "build"
 pages:
   stage: deploy
   dependencies:
-    - build_docs
+    - build_html
+    - build_jsonld
   script:
     - mv build/ public/
   artifacts:

Dataset/p65_example_data.yaml

-owner: "Kermit the Frog"
-type: raw
-principalInvestigator: "Dr. Bunsen Honeydew"
+datasetName: "P65 XAS"
+description: "XAS dataset."
+principalInvestigator: "Dr. Anjali"
+owner: "Anjali"
+contactEmail: "anjali@example.com"
+sourceFolder: "/tmp"
+size: 123456
+creationLocation: "Hamburg"
+creationTime: "2024-06-26T12:00:00Z"
+type: "raw"
+instrumentId: "Instrument001"
+proposalId: "20010161"
+dataFormat: "HDF5"
+isPublished: False
+ownerGroup: "hasylab"
+accessGroups: ["FS-EC"]
+keywords: ["XAS", "data", "keywords"]
+license: "CC BY 4.0"
 scientificMetadata:
-#   special_p65_parameter: "something"
+      experimentID: EXP-001
+      sample:
+            sampleID: SAMPLE-001
+            material: Platinum on Alumina
+            preparationMethod: Impregnation
+            catalyst: Platinum (1 wt%)
+            reactionConditions: 
+              temperature:
+                  unit: degC
+                  value: 300
+              pressure:
+                  unit: atm
+                  value: 1
+              gas: H2
+            description: A sample of platinum nanoparticles dispersed on gamma-alumina support.
+            dSpacing: 3.54  # in angstroms
+            crystalType: Cubic
+      instrument:
+            instrumentID: INSTR-001
+            name: Synchrotron Beamline Instrument
+            facility: DESY
+            beamline: P65
+            energyRange:
+              minEnergy: 
+                  value: 5000
+                  unit: eV
+              maxEnergy:
+                  value: 25000
+                  unit: eV  # in eV
+      technique:
+            techniqueType: EXAFS
+            edge: K-edge
+            detector: Fluorescence
+            monochromator: Si111
+      dataProcessing:
+            software: Athena
+            version: 0.9.26
+            method: Background subtraction and Fourier transform
+      metadata:
+            project: "Catalysis Study for Platinum-Based Catalysts"
+            experimenter: "Dr. John Doe"
+      startTime: "2024-08-01T09:00:00Z"
+      endTime: "2024-08-01T12:00:00Z"
 
-  rockit_data:
-   #  DiffractionMetadata:
-   sample_environment:
-      GasComposition: "CO2"
-      GasSystem:
-         GasSys_Instrument_type: massflow_meter
-         GasSys_Instrument_type: flow_calibrator
-         # Instruments:  
-         #   Instrument_id: [25,35] 
-      motors: 
-         motor1:
-            value: 5
-            unit: mm
-         motor2:
-            value: 5
-            unit: mm
-         motor3:
-            value: 5
-            unit: mm
-        
-       
 
-  sample:
-    temperature:
-       value: 15
-       unit: K
-   #   layer_structure: "Si/SiO/DIP"
-    chemical_composition: "Si/SiO/DIP"
\ No newline at end of file

schema/base_sample.yaml

@@ -13,18 +13,35 @@ classes:
     abstract: true
     description: generic sample description
     attributes:
+      sampleID:
+        description: Unique identifier for the sample
+        required: true
+      material:
+        description: Material of the sample
+      preparationMethod:
+        description: Method used to prepare the sample
+      catalyst:
+        description: Catalyst used in the experiment
+      reactionConditions:
+        description: Reaction conditions (e.g., temperature, pressure)
+        range: ReactionConditions
+      description:
+        description: Detailed description of the sample
+      dSpacing:
+        description: Interplanar spacing (d-spacing) of the sample in angstroms
+        range: float
       chemical_composition:
         range: string
-      temperature:
-        range: TemperatureMeasurment
-      pressure:
-        range: PressureMeasurement
       environment:
         description: indentifier of specific sample environment
         range: string
         # Todo: see if this is feasable ... maybe sample envirenment should be split from sample completely
   SolidStateSample:
     is_a: Sample
+    attributes:
+      crystalType:
+        description: Crystal structure type of the sample (e.g., cubic, hexagonal)
+        range: CrystalTypeEnum
   LiquidSurfaceSammple:
     is_a: Sample
     attributes:
@@ -36,3 +53,27 @@ classes:
       layer_structure:
         range: string
         # Todo: see if feasable: use ORSO SampleModel String
+  ReactionConditions:
+    description: Reaction conditions (e.g., temperature, pressure)
+    attributes: 
+      temperature: 
+        range: TemperatureMeasurment
+      pressure: 
+        range: PressureMeasurement 
+      gas: 
+        range: string
+enums:
+  CrystalTypeEnum:
+    permissible_values:
+      Cubic:
+        description: Cubic crystal structure
+      Hexagonal:
+        description: Hexagonal crystal structure
+      Tetragonal:
+        description: Tetragonal crystal structure
+      Orthorhombic:
+        description: Orthorhombic crystal structure
+      Monoclinic:
+        description: Monoclinic crystal structure
+      Triclinic:
+        description: Triclinic crystal structure
\ No newline at end of file

schema/base_scicat_dataset.yaml

@@ -7,7 +7,6 @@ prefixes:
 imports: [linkml:types]
 # default_prefix: linkml
 default_range: string
-
 classes:
   ScicatDataset:
     abstract: true
@@ -116,6 +115,7 @@ classes:
         description: The ID of the proposal to which the dataset belongs.
       isPublished:
         description: The ID of the proposal to which the dataset belongs.
+        range: boolean
       #~ sampleId:
         #~ description: ID of the sample used when collecting the data.
       instrumentId:
@@ -124,7 +124,9 @@ classes:
         description: Defines the group which owns the data, and therefore has unrestricted access to this data. Usually a pgroup like p12151
       accessGroups: 
         description: Defines the group which owns the data, and therefore has unrestricted access to this data. Usually a pgroup like p12151
+        multivalued: true
     #   #~ instrumentGroup: {}
+      
 
 
   SciCatProposal:
@@ -196,20 +198,20 @@ classes:
   MetaObject:
     class_uri: linkml:Any
 
-  Instrument:
-    abstract: true
-    attributes:
-      pid:
-        description: PID of the instrument
+  # Instrument:
+  #   abstract: true
+  #   attributes:
+  #     pid:
+  #       description: PID of the instrument
       
-      uniqueName: 
-        description: The unique name of the instrument. This name has to be unique within the scicat instance.
+  #     uniqueName: 
+  #       description: The unique name of the instrument. This name has to be unique within the scicat instance.
       
-      name: 
-        description: The common name of the instrument. This name can be non unique as it is the name that users use to commonly refer to this instrument.
+  #     name: 
+  #       description: The common name of the instrument. This name can be non unique as it is the name that users use to commonly refer to this instrument.
       
-      customMetadata:
-         range: MetaObject
+  #     customMetadata:
+  #        range: MetaObject
 
 
 enums:

schema/beamline_p65.yaml → schema/beamline_p65_xas.yaml

@@ -18,19 +18,23 @@ imports:
   - base_scicat_dataset
   - base_sample
   - rockit
+  - technique_xas_catalysis
+  
 default_range: string
 classes:
   P65ScientificMetadata:
     description: The metadata associated with P65
     is_a: PetraScientificMetadata
-    mixins: [rockit_Container]
-    attributes:
-      # special_p65_parameter:
-      #   description: something that only exists at P65
-      catalyst: 
-        description: Name of the catalyst used for experiment
-      sample:
-        range: SolidStateSample 
+    # mixins: [rockit_Container]
+    mixins: [CatalysisExperiment]
+    # attributes:
+    #   # special_p65_parameter:
+    #   #   description: something that only exists at P65
+    #   catalyst: 
+    #     description: Name of the catalyst used for experiment
+    #   sample:
+    #     range: SolidStateSample 
+
   P65Dataset:
     tree_root: true
     is_a: ScicatDataset

schema/rockit.yaml

@@ -11,11 +11,11 @@ default_range: string
 classes:
 
   rockit:
-    description: Metadata fo rockit
+    description: Metadata for rockit
     # should get all what one can get from PaNET
     attributes:
       sample_environment:
-       range: sample_environment
+       range: SampleEnvironment
       # GasComposition:
       #   description: Describe gas composition
       #   range: string

schema/xas_catalysis.yaml → schema/technique_xas_catalysis.yaml

-id: https://example.org/xas-catalysis
-name: XAS Catalysis
-description: Schema for X-ray Absorption Spectroscopy (XAS) technique in catalysis experiments at the DESY P64 beamline
+id: https://public-data.desy.de/metadata/xas
+name: technique_xas_catalysis
+description: Schema for X-ray Absorption Spectroscopy (XAS) technique in catalysis experiments at the DESY P65 beamline
 prefixes:
   schema: http://schema.org/
   linkml: https://w3id.org/linkml/
@@ -8,25 +8,40 @@ default_range: string
 
 imports:
   - linkml:types
+  - base_types
+  - base_sample
+  - base_scicat_dataset
+  # - base_catalogue_metadata
+
+
 
-default_prefix: xas_catalysis
 
-types:
-  float:
-    base: float
-  datetime:
-    base: datetime
 
 classes:
+
+  # P65ScientificMetadata:
+  #   description: The metadata associated with P65
+  #   is_a: PetraScientificMetadata
+  #   mixins: [CatalysisExperiment]
+
+  # P65Dataset:
+  #   tree_root: true
+  #   is_a: ScicatDataset
+  #   description: The metadata associated with P65
+  #   attributes:
+  #     scientificMetadata:
+  #       range: P65ScientificMetadata
+  #       required: true
+
   CatalysisExperiment:
-    description: Information about the catalysis experiment using XAS at the DESY P64 beamline
+    description: Information about the catalysis experiment using XAS at the DESY P65 beamline
     attributes:
       experimentID:
         description: Unique identifier for the experiment
         required: true
       sample:
         description: Information about the sample used in the experiment
-        range: Sample
+        range: SolidStateSample
       instrument:
         description: Information about the instrument used for the experiment
         range: Instrument
@@ -46,23 +61,6 @@ classes:
         description: End time of the experiment
         range: datetime
 
-  Sample:
-    description: Information about the sample used in the XAS catalysis experiment
-    attributes:
-      sampleID:
-        description: Unique identifier for the sample
-        required: true
-      material:
-        description: Material of the sample
-      preparationMethod:
-        description: Method used to prepare the sample
-      catalyst:
-        description: Catalyst used in the experiment
-      reactionConditions:
-        description: Reaction conditions (e.g., temperature, pressure)
-      description:
-        description: Detailed description of the sample
-
   Instrument:
     description: Information about the instrument used for the XAS catalysis experiment
     attributes:
@@ -73,10 +71,10 @@ classes:
         description: Name of the instrument
       facility:
         description: Facility where the instrument is located
-        default: DESY
+        range: string
       beamline:
         description: Beamline used for the experiment
-        default: P64
+        range: string
       energyRange:
         description: Energy range used in the experiment
         range: EnergyRange
@@ -90,21 +88,16 @@ classes:
         description: Version of the software
       method:
         description: Method or algorithm used for data processing
-
-  MetaObject:
-    description: Generic metadata object
-    class_uri: linkml:Any
-
   EnergyRange:
     description: Energy range used in the XAS experiment
     attributes:
       minEnergy:
         description: Minimum energy in electron volts (eV)
-        range: float
+        range: EnergyMeasurement
         required: true
       maxEnergy:
         description: Maximum energy in electron volts (eV)
-        range: float
+        range: EnergyMeasurement
         required: true
 
   Technique:
@@ -158,3 +151,5 @@ enums:
         description: Silicon (311) monochromator
       Ge111:
         description: Germanium (111) monochromator
+
+  

scripts/gen_allformat.py

+import argparse
+import os
+
+from linkml.generators.jsonschemagen import JsonSchemaGenerator 
+from linkml.generators.jsonldgen import JSONLDGenerator
+from linkml.generators.owlgen import OwlSchemaGenerator
+def convert_linkml_to_jsonschema(yaml_schema_path, json_schema_output):
+    # Load the schema using SchemaView
+    
+    json_schema = JsonSchemaGenerator(yaml_schema_path).serialize()
+    json_ld = JSONLDGenerator(yaml_schema_path).serialize()
+    owl_shema = OwlSchemaGenerator(yaml_schema_path).serialize()
+    # Save JSON schema to file
+    file_name=json_schema_output.split(".")[0]
+    with open(f'{file_name}.json', 'w') as file:
+        file.write(json_schema)
+    with open(f'{file_name}.jsonld', 'w') as file:
+        file.write(json_ld)
+    with open(f'{file_name}.owl', 'w') as file:
+        file.write(owl_shema)
+
+def main():
+    parser = argparse.ArgumentParser(description="Convert a LinkML YAML schema to JSON schema.")
+    parser.add_argument('-i', '--input', type=str, default='scicat/scicat.yaml', help='The path to the LinkML YAML schema file to be read (default: schemas/schema.yaml).')
+    parser.add_argument('-o', '--output', type=str, default='scicat/output/scicat.json', help='The path to save the JSON schema output (default: output/schema.json).')
+
+    args = parser.parse_args()
+
+    # Ensure output directory exists
+    os.makedirs(os.path.dirname(args.output), exist_ok=True)
+
+    # Convert LinkML YAML schema to JSON schema
+    convert_linkml_to_jsonschema(args.input, args.output)
+    print(f"LinkML YAML schema successfully converted and saved to {args.output}")
+
+if __name__ == "__main__":
+    main()
+
+
+
+# import yaml
+# import argparse
+# import json
+# from linkml_runtime.utils.schemaview import SchemaView
+# from linkml_runtime.linkml_model.meta import SchemaDefinition
+
+# # from linkml.generators.jsonschemagen import 
+# from linkml.generators.jsonschemagen import JsonSchemaGenerator
+
+# def convert_linkml_to_jsonschema(yaml_schema_path, json_schema_output):
+#     # Load the schema using SchemaView
+#     schema_view = SchemaView(yaml_schema_path)
+    
+
+#     json_schema = JsonSchemaGenerator(schema_view.schema).serialize()
+    
+#     # Save JSON schema to file
+#     with open(json_schema_output, 'w') as file:
+#         file.write(json_schema)
+# def main():
+#     # Initialize the parser
+#     parser = argparse.ArgumentParser(description="Gen json schema")
+
+#     # Add the file argument
+#     parser.add_argument('-i', '--input', type=str, default='schema.yaml', help='The path to the LinkML YAML schema file to be read (default: schema.yaml).')
+#     parser.add_argument('-o', '--output', type=str, default='schema.json', help='The path to save the JSON schema output (default: schema.json).')
+#     # Parse the arguments
+#     args = parser.parse_args()
+
+#     # Read and print the file content
+#     schema_view = convert_linkml_to_jsonschema(args.input,args.output)
+#     # json_schema = JsonSchemaGenerator(schema_view.schema).serialize()
+#     # with open("file.json", "w") as outfile: 
+#     #     json.dump(json_schema, outfile)
+
+# if __name__ == "__main__":
+#     main()
+

scripts/validate_p65_data.sh

+# linkml-validate -s ./schema/beamline_p08.yaml -C P08Dataset  ./Dataset/p08_example_data.yaml   
+linkml-validate -s ../schema/beamline_p65.yaml -C P65Dataset ../Dataset/p65_example_data.yaml 
\ No newline at end of file