Created availability to send url link for parsing

endpoints/__init__.py

@@ -4,5 +4,5 @@ from .user import get_me, create_bot_token, get_bot_tokens, delete_bot_token
 from .parser import load_report
 from .indicators import get_indicators_from_group, get_indicator_groups
 from .watcher import create_watcher
-from .parser import get_reports
+from .parser import get_reports, get_page_report
 from .static import token

endpoints/parser.py

@@ -2,6 +2,7 @@ import os
 from uuid import uuid4
 from tempfile import NamedTemporaryFile
 
+from aiohttp import request as aiohttp_request
 from starlette.background import BackgroundTasks
 from starlette.datastructures import UploadFile
 from starlette.requests import Request
@@ -11,8 +12,8 @@ from models import Report, IndicatorGroup
 from models.indicator import IndicatorGroupPD
 from models.report import ReportPD
 from parsers.pdf_parser import process_pdf
-from parsers.text_parser import CollectedData
-from responses.errors import ReportNotPresented
+from parsers.text_parser import CollectedData, find_ioc
+from responses.errors import ReportNotPresented, ReportURLError
 from responses.responses import OkResponse
 
 
@@ -50,3 +51,21 @@ async def get_reports(request: Request):
         report.owner = request.state.user
         as_pd.append(ReportPD.from_orm(report))
     return OkResponse(as_pd)
+
+
+async def get_page_report(request: Request):
+    """ Creates group from news page """
+    url = request.query_params["url"]
+    try:
+        async with aiohttp_request("GET", url) as resp:
+            page = await resp.text()
+            if resp.status != 200:
+                raise ReportURLError
+    except Exception:
+        raise ReportURLError
+    data = find_ioc(page.lower())
+    group = await IndicatorGroup.from_reports_collected_data(data, request.state.user)
+    group.description = "Group from page " + url
+    await group.save()
+    group_pd = IndicatorGroupPD.from_orm(group)
+    return OkResponse({"indicator_group": group_pd})

endpoints/routes.py

@@ -20,7 +20,8 @@ api_routes = [
     Route("/deleteBotToken", delete_bot_token, methods=["GET"]),
     Route("/createWatcher", create_watcher, methods=["POST"]),
     Route("/getIndicatorGroups", get_indicator_groups, methods=["GET"]),
-    Route("/getReports", get_reports, methods=["GET"])
+    Route("/getReports", get_reports, methods=["GET"]),
+    Route("/loadPageReport", get_page_report, methods=["GET"]),
 ]
 
 admin_routes = []

responses/errors.py

@@ -87,3 +87,10 @@ class TelegramInvalidLink(ApiError):
     code = 604
     description = "Telegram channel join link is invalid"
     http_code = 400
+
+
+class ReportURLError(ApiError):
+    """ Report URL is invalid """
+    code = 701
+    description = "Report URL is invalid, or 200 status code is not returned"
+    http_code = 422

tests/test_indicator.py

@@ -107,3 +107,33 @@ def test_reports_endpoint(sti_auth, user, mts_report):
     assert 1 == len(resp_reports.json()["data"])
     print(resp_reports.json())
     assert resp_reports.json()["data"][0]["id"] == report_id
+
+
+def test_parse_link_report_empty(sti_auth):
+    test_url = "https://storage.yandexcloud.net/ivanprogramming/empty_report.html"
+
+    resp = sti_auth.get(f"/api/loadPageReport?url={test_url}")
+    assert resp.status_code == 200
+
+    data = resp.json()["data"]
+    report_id = data["indicator_group"]["id"]
+
+    resp = sti_auth.get(f"/api/getIndicatorsFromGroup?group_id={report_id}")
+    assert resp.status_code == 200
+
+    assert len(resp.json()["data"]["indicators"]) == 0
+
+
+def test_parse_link_report(sti_auth):
+    test_url = "https://storage.yandexcloud.net/ivanprogramming/report.html"
+
+    resp = sti_auth.get(f"/api/loadPageReport?url={test_url}")
+    assert resp.status_code == 200
+
+    data = resp.json()["data"]
+    report_id = data["indicator_group"]["id"]
+
+    resp = sti_auth.get(f"/api/getIndicatorsFromGroup?group_id={report_id}")
+    assert resp.status_code == 200
+
+    assert len(resp.json()["data"]["indicators"]) == 3