CSCMETAX-280:[ADD] datacatalog harvesting

Author: jkesanie

src/metax_api/api/oaipmh/base/metax_oai_server.py

@@ -7,7 +7,7 @@
 from oaipmh.error import IdDoesNotExistError
 from oaipmh.error import BadArgumentError
 
-from metax_api.models.catalog_record import CatalogRecord
+from metax_api.models.catalog_record import CatalogRecord, DataCatalog
 from metax_api.services import CatalogRecordService as CRS
 
 syke_url_prefix_template = 'http://metatieto.ymparisto.fi:8080/geoportal/catalog/search/resource/details.page?uuid=%s'
@@ -16,7 +16,7 @@
 class MetaxOAIServer(ResumptionOAIPMH):
 
     def _is_valid_set(self, set):
-        if not set or set == 'urnresolver' or set in settings.OAI['SET_MAPPINGS']:
+        if not set or set in ['urnresolver', 'datacatalogs'] or set in settings.OAI['SET_MAPPINGS']:
             return True
         return False
 
@@ -32,16 +32,20 @@ def _get_filtered_records(self, set, cursor, batch_size, from_=None, until=None)
         if not self._is_valid_set(set):
             raise BadArgumentError('invalid set value')
 
-        query_set = CatalogRecord.objects.all()
+        proxy = CatalogRecord
+        if set == 'datacatalogs':
+            proxy = DataCatalog
+
+        query_set = proxy.objects.all()
         if from_ and until:
-            query_set = CatalogRecord.objects.filter(date_modified__gte=from_, date_modified__lte=until)
+            query_set = proxy.objects.filter(date_modified__gte=from_, date_modified__lte=until)
         elif from_:
-            query_set = CatalogRecord.objects.filter(date_modified__gte=from_)
+            query_set = proxy.objects.filter(date_modified__gte=from_)
         elif until:
-            query_set = CatalogRecord.objects.filter(date_modified__lte=until)
+            query_set = proxy.objects.filter(date_modified__lte=until)
 
         if set:
-            if set == 'urnresolver':
+            if set in ['urnresolver', 'datacatalogs']:
                 pass
             else:
                 query_set = query_set.filter(
@@ -98,72 +102,76 @@ def _get_oaic_dc_value(self, value, lang=None):
             valueDict['lang'] = lang
         return valueDict
 
-    def _get_oai_dc_metadata(self, record):
-        identifier = self._get_oaic_dc_value(record.research_dataset.get('preferred_identifier'))
+    def _get_oai_dc_metadata(self, record, json, type):
+        identifier = []
+        if 'preferred_identifier' in json:
+            identifier.append(self._get_oaic_dc_value(json.get('preferred_identifier')))
+        if 'identifier' in json:
+            identifier.append(self._get_oaic_dc_value(json.get('identifier')))
 
         title = []
-        title_data = record.research_dataset.get('title', {})
+        title_data = json.get('title', {})
         for key, value in title_data.items():
             title.append(self._get_oaic_dc_value(value, key))
 
         creator = []
-        creator_data = record.research_dataset.get('creator', [])
+        creator_data = json.get('creator', [])
         for value in creator_data:
             if 'name' in value:
                 creator.append(self._get_oaic_dc_value(value.get('name')))
 
         subject = []
-        subject_data = record.research_dataset.get('keyword', [])
+        subject_data = json.get('keyword', [])
         for value in subject_data:
             subject.append(self._get_oaic_dc_value(value))
-        subject_data = record.research_dataset.get('field_of_science', [])
+        subject_data = json.get('field_of_science', [])
         for value in subject_data:
             for key, value2 in value.get('pref_label', {}).items():
                 subject.append(self._get_oaic_dc_value(value2, key))
-        subject_data = record.research_dataset.get('theme', [])
+        subject_data = json.get('theme', [])
         for value in subject_data:
             for key, value2 in value.get('pref_label', {}).items():
                 subject.append(self._get_oaic_dc_value(value2, key))
 
         desc = []
-        desc_data = record.research_dataset.get('description', [])
+        desc_data = json.get('description', [])
         for value in desc_data:
             for key, value2 in value.items():
                 desc.append(self._get_oaic_dc_value(value2, key))
 
         publisher = []
-        publisher_data = record.research_dataset.get('publisher', {})
+        publisher_data = json.get('publisher', {})
         for key, value in publisher_data.get('name', {}).items():
             publisher.append(self._get_oaic_dc_value(value))
 
         contributor = []
-        contributor_data = record.research_dataset.get('contributor', [])
+        contributor_data = json.get('contributor', [])
         for value in contributor_data:
             if 'name' in value:
                 contributor.append(self._get_oaic_dc_value(value.get('name')))
 
         date = self._get_oaic_dc_value(str(record.date_created))
 
         language = []
-        language_data = record.research_dataset.get('language', [])
+        language_data = json.get('language', [])
         for value in language_data:
             for key, value2 in value.items():
                 language.append(self._get_oaic_dc_value(value2))
 
         relation = []
-        relation_data = record.research_dataset.get('relation', [])
+        relation_data = json.get('relation', [])
         for value in relation_data:
             if 'identifier'in value.get('entity', {}):
                 relation.append(self._get_oaic_dc_value(value['entity']['identifier']))
 
         coverage = []
-        coverage_data = record.research_dataset.get('spatial', [])
+        coverage_data = json.get('spatial', [])
         for value in coverage_data:
             if 'geographic_name' in value:
                 coverage.append(self._get_oaic_dc_value(value['geographic_name']))
 
         rights = []
-        rights_data = record.research_dataset.get('access_rights', {})
+        rights_data = json.get('access_rights', {})
         for value in rights_data.get('description', []):
             for key, value2 in value.items():
                 rights.append(self._get_oaic_dc_value(value2, key))
@@ -172,10 +180,10 @@ def _get_oai_dc_metadata(self, record):
                 rights.append(self._get_oaic_dc_value(value['identifier']))
 
         types = []
-        types.append(self._get_oaic_dc_value('Dataset'))
+        types.append(self._get_oaic_dc_value(type))
 
         meta = {
-            'identifier':  [identifier],
+            'identifier':  identifier,
             'title': title,
             'creator': creator,
             'subject': subject,
@@ -202,16 +210,16 @@ def _get_oai_datacite_metadata(self, record):
         }
         return meta
 
-    def _get_metadata_for_record(self, record, metadata_prefix):
+    def _get_metadata_for_record(self, record, json, type, metadata_prefix):
         meta = {}
 
         # strip sensitive fields from research_dataset. note: the modified research_dataset
         # is placed back into the record's research_dataset -field. meaning, an accidental call
         # of record.save() would overwrite the original data
-        record.research_dataset = CRS.strip_catalog_record(record.research_dataset)
+        record.research_dataset = CRS.strip_catalog_record(json)
 
         if metadata_prefix == 'oai_dc':
-            meta = self._get_oai_dc_metadata(record)
+            meta = self._get_oai_dc_metadata(record, json, type)
         elif metadata_prefix == 'oai_datacite':
             meta = self._get_oai_datacite_metadata(record)
         elif metadata_prefix == 'oai_dc_urnresolver':
@@ -226,9 +234,14 @@ def _get_header_timestamp(self, record):
             timestamp = record.date_created
         return timezone.make_naive(timestamp)
 
-    def _get_oai_item(self, record, metadata_prefix):
-        identifier = record.identifier
-        metadata = self._get_metadata_for_record(record, metadata_prefix)
+    def _get_oai_item(self, identifier, record, metadata_prefix):
+        metadata = self._get_metadata_for_record(record, record.research_dataset, 'Dataset', metadata_prefix)
+        item = (common.Header('', identifier, self._get_header_timestamp(record), ['metax'], False),
+                common.Metadata('', metadata), None)
+        return item
+
+    def _get_oai_catalog_item(self, identifier, record, metadata_prefix):
+        metadata = self._get_metadata_for_record(record, record.catalog_json, 'Datacatalog', metadata_prefix)
         item = (common.Header('', identifier, self._get_header_timestamp(record), ['metax'], False),
                 common.Metadata('', metadata), None)
         return item
@@ -281,18 +294,24 @@ def listMetadataFormats(self, identifier=None):
 
     def listSets(self, cursor=None, batch_size=None):
         """Implement OAI-PMH verb ListSets."""
-        data = []
+        data = [('datacatalogs', 'datacatalog', '')]
         for set_key in settings.OAI['SET_MAPPINGS'].keys():
             data.append((set_key, set_key, ''))
         return data
 
+    def _get_record_identifier(self, record, set):
+        if set == 'datacatalogs':
+            return record.catalog_json['identifier']
+        else:
+            return record.identifier
+
     def listIdentifiers(self, metadataPrefix=None, set=None, cursor=None,
                         from_=None, until=None, batch_size=None):
         """Implement OAI-PMH verb listIdentifiers."""
         records = self._get_filtered_records(set, cursor, batch_size, from_, until)
         data = []
         for record in records:
-            identifier = record.research_dataset.get('preferred_identifier')
+            identifier = self._get_record_identifier(record, set)
             data.append(common.Header('', identifier, self._get_header_timestamp(record), ['metax'], False))
         return data
 
@@ -302,18 +321,30 @@ def listRecords(self, metadataPrefix=None, set=None, cursor=None, from_=None,
         data = []
         records = self._get_filtered_records(set, cursor, batch_size, from_, until)
         for record in records:
-            data.append(self._get_oai_item(record, metadataPrefix))
+            identifier = self._get_record_identifier(record, set)
+            if set == 'datacatalogs':
+                data.append(self._get_oai_catalog_item(identifier, record, metadataPrefix))
+            else:
+                data.append(self._get_oai_item(identifier, record, metadataPrefix))
         return data
 
     def getRecord(self, metadataPrefix, identifier):
         """Implement OAI-PMH verb GetRecord."""
         try:
-            record = CatalogRecord.objects.get(
-                # data_catalog__catalog_json__identifier__in=self._get_default_set_filter(),
-                identifier__exact=identifier
-            )
+            if CatalogRecord.objects.filter(identifier__exact=identifier).exists():
+                record = CatalogRecord.objects.get(identifier__exact=identifier)
+                json = record.research_dataset
+                type = 'Dataset'
+            else:
+                record = DataCatalog.objects.get(catalog_json__identifier__exact=identifier)
+                json = record.catalog_json
+                type = 'Datacatalog'
+
         except CatalogRecord.DoesNotExist:
             raise IdDoesNotExistError("No dataset with id %s available through the OAI-PMH interface." % identifier)
-        metadata = self._get_metadata_for_record(record, metadataPrefix)
+        except DataCatalog.DoesNotExist:
+            raise IdDoesNotExistError("No datacatalog with id %s available through the OAI-PMH interface." % identifier)
+
+        metadata = self._get_metadata_for_record(record,  json, type, metadataPrefix)
         return (common.Header('', identifier, self._get_header_timestamp(record), ['metax'], False),
                 common.Metadata('', metadata), None)

src/metax_api/tests/api/oaipmh/__init__.py

@@ -0,0 +1,2 @@
+from .minimal_api import *
+from .syke import *