Crawling JavaScript inserted json-ld

.gitignore

@@ -2,3 +2,4 @@
 *.pyc
 *.swp
 */__pycache__/
+data/*.db

bioschemas/extractors.py

@@ -1,61 +1,41 @@
-import logging
-import os
-
-import bs4
-import json
-import requests
-from requests_testadapter import Resp
-
-logger = logging.getLogger(__name__)
-
-
-class LocalFileAdapter(requests.adapters.HTTPAdapter):
-    def build_response_from_file(self, request):
-        file_path = request.url[7:]
-        with open(file_path, 'rb') as file:
-            buff = bytearray(os.path.getsize(file_path))
-            file.readinto(buff)
-            resp = Resp(buff)
-            r = self.build_response(request, resp)
-
-            return r
-
-    def send(self, request, stream=False, timeout=None,
-             verify=True, cert=None, proxies=None):
-
-        return self.build_response_from_file(request)
-
-
-class ExtractorFromHtml:
-    def __init__(self, config):
-        self.config = config
-
-    def extract_jsonld_from_url(self, url):
-        """
-        Extract jsonld from the given url
-
-        :param url:
-        :return: [<jsonld>+]
-        """
-        requests_session = requests.session()
-        requests_session.mount('file://', LocalFileAdapter())
-        r = requests_session.get(url)
-        return self._extract_jsonld_from_html(r.text)
-
-    def _extract_jsonld_from_html(self, html):
-        """
-        Extract jsonld from html
-
-        :param html:
-        :return: [<jsonld>]
-        """
-        soup = bs4.BeautifulSoup(html, 'html.parser')
-        ldjson_script_sections = soup.find_all('script', type='application/ld+json')
-        logger.debug('Found %d ld+json sections', len(ldjson_script_sections))
-
-        jsonlds = []
-
-        for ldjson_script_section in ldjson_script_sections:
-            jsonlds.append(json.loads(ldjson_script_section.string))
-
-        return jsonlds
+import logging
+
+from requests_html import HTMLSession
+import json
+
+logger = logging.getLogger(__name__)
+
+
+class ExtractorFromHtml:
+    def __init__(self, config):
+        self.config = config
+
+    def extract_jsonld_from_url(self, url):
+        """
+        Extract jsonld from the given url
+        :param url:
+        :return: [<jsonld>+]
+        """
+        html_response = self.get_html_from_url(url)
+        return self.extract_jsonld_from_html(html_response)
+
+    @staticmethod
+    def get_html_from_url(url, is_dynamic=False):
+        with HTMLSession() as session:
+            response = session.get(url)
+            if is_dynamic:
+                response.html.render()
+            return response.html
+
+    @staticmethod
+    def extract_jsonld_from_html(html):
+        """
+        Extract jsonld from html
+
+        :param html: request-html HTML object
+        :return: [<jsonld>]
+        """
+        ldjson_script_sections = html.find("script[type='application/ld+json']")  # use css selector to find the tag
+        logger.debug('Found %d ld+json sections', len(ldjson_script_sections))
+
+        return [json.loads(section.text.replace('\\n', ''), strict=False) for section in ldjson_script_sections]

bioschemas/test_extractors.py

@@ -1,4 +1,7 @@
 import unittest
+import json
+
+from requests_html import HTML
 
 import bioschemas
 from bioschemas.extractors import ExtractorFromHtml
@@ -7,6 +10,14 @@
 
 
 class TestExtractors(unittest.TestCase):
+    TEST_URL = 'https://httpbin.org/anything'
+
+    def test_get_html_from_url(self):
+        e = ExtractorFromHtml(config)
+        html_response = e.get_html_from_url(self.TEST_URL)
+        json_content = json.loads(html_response.raw_html.decode())
+        self.assertEqual(json_content['url'], self.TEST_URL)
+
     def test_jsonld_extraction_from_html(self):
         html = '''<script type="application/ld+json">
         {
@@ -21,7 +32,7 @@ def test_jsonld_extraction_from_html(self):
         '''
 
         e = ExtractorFromHtml(config)
-        jsonlds = e._extract_jsonld_from_html(html)
+        jsonlds = e.extract_jsonld_from_html(HTML(html=html))
         self.assertEqual(len(jsonlds), 1)
 
         jsonld = jsonlds[0]