Merge branch 'features/config'

confs/api_client_1mail.ini

@@ -0,0 +1,9 @@
+[probe]
+identifiant_sonde =  dummy
+nom_sonde =  dummy
+emails = 1@1
+
+
+
+
+

confs/api_client_Alexi.ini

@@ -0,0 +1,8 @@
+
+[probe]
+identifiant_sonde = f570d2a9-f46a-4068-9e74-8543f1417bbb
+nom_sonde =  Alexi
+emails = colin.goutte@free.fr
+        alexi@savediffusion.fr
+
+

confs/api_client_Colin.ini

@@ -0,0 +1,7 @@
+
+[probe]
+identifiant_sonde = 04e5fa0f-d90c-424d-9fff-41147762ecbb
+nom_sonde =  Colin
+emails = colin.goutte@free.fr
+
+

confs/api_client_test_sonde.ini

@@ -0,0 +1,10 @@
+[probe]
+identifiant_sonde = 838266b2-fc3a-4430-95e8-f7f0d0fc9871
+nom_sonde =  SondeTest
+emails = 1@1
+         2@2
+
+
+
+
+

confs/make_api.py

@@ -0,0 +1,22 @@
+from uuid import uuid4
+
+nom_sonde = input("Choisir un nom pour la sonde:")
+
+
+content = """
+[probe]
+identifiant_sonde = {id_sonde}
+nom_sonde =  {nom_sonde}
+emails = {emails}
+
+"""
+
+
+with open(f"api_client_{nom_sonde}.ini", "w") as f:
+    f.write(
+        content.format(
+            id_sonde=str(uuid4()),
+            nom_sonde=nom_sonde,
+            emails="\n         ".join(["aaa", "bbb"]),
+        )
+    )

confs/relais_test_sonde.ini

@@ -0,0 +1,7 @@
+[relais]
+identitifant_sonde = 838266b2-fc3a-4430-95e8-f7f0d0fc9871
+api_login_url = 'https://merlin.savediffusion.fr/api/1.1/user/login'
+api_status_url = 'https://merlin.savediffusion.fr/api/1.1/status"
+api_user_login = 'admin'
+api_user_password = 'Watermark35'
+forward_api_address = 'https:/papi.silib.re/'

papi/config.py

@@ -0,0 +1,29 @@
+import os
+import glob
+
+import configparser
+
+
+def read_config(*, pattern="*"):
+
+    paths = []
+    for relpath in glob.glob(f"confs/{pattern}.ini"):
+        if "/api_client" in relpath:
+            paths.append(os.path.abspath(relpath))
+    res = []
+    for path in paths:
+        conf = configparser.ConfigParser()
+        conf.read(path)
+        probe_settings = dict(conf["probe"])
+        emails = probe_settings["emails"]
+        stripped = [x.strip() for x in emails.split()]
+        probe_settings["emails"] = stripped
+        probe_settings["debug_source_path"] = path
+        res.append(probe_settings)
+
+    return res
+
+
+def apply_config(db, settings: list[dict]):
+
+    pass

papi/mail_sendermodel.py

@@ -4,16 +4,16 @@ import os
 from papi.credentials import api_key, api_secret
 
 
-def sendmail(htmlpart):
+def sendmail(htmlpart, emails, textpart="", subject="Monitorig api"):
 
     mailjet = Client(auth=(api_key, api_secret), version="v3.1")
     data = {
         "Messages": [
             {
                 "From": {"Email": "colin.goutte@free.fr", "Name": "Colin"},
-                "To": [{"Email": "colin.goutte@free.fr", "Name": "Colin"}],
-                "Subject": "Monitoring API",
-                "TextPart": "My first Mailjet email",
+                "To": [{"Email": email} for email in emails],
+                "Subject": "Monitoring API" + subject,
+                "TextPart": "" + textpart,
                 "HTMLPart": "" + htmlpart,
                 "CustomID": "AppGettingStartedTest",
             }

papi/main.py

@@ -1,5 +1,7 @@
 from typing import Optional, List
 from configparser import ConfigParser
+import contextlib
+import logging
 
 from pydantic import BaseModel
 
@@ -14,6 +16,8 @@ from . import utils
 from papi.sqlapp.database import Base, SessionLocal, engine
 from papi.sqlapp import crud
 from papi.sqlapp import schemas
+from papi.config import read_config
+
 
 app = FastAPI()
 
@@ -28,14 +32,29 @@ Base.metadata.create_all(bind=engine)
 conf = ConfigParser()
 conf.read("conf_notifications.ini")
 
+configurations = read_config(pattern="*api_client*")
 
-def sondeid2notifsemails(idsonde, mapping=conf):
+
+logger = logging.getLogger()
+console = logging.StreamHandler()
+console.setLevel(logging.INFO)
+logger.setLevel(logging.INFO)
+logger.addHandler(console)
+
+
+PROBES = {}
+
+
+def sondeid2notifsemails(idsonde, mapping=None):
+    if mapping is None:
+        mapping = PROBES
     try:
+        breakpoint()
         section = mapping[idsonde]
     except KeyError:  # pragma: no cover
         raise KeyError(f"{idsonde} non trouvé dans {list(mapping.keys())}")
 
-    mails = section["email"]
+    mails = section["emails"]
     if isinstance(mails, str):
         mails = [x.strip() for x in mails.split("\n")]
     return mails
@@ -49,6 +68,38 @@ def get_db():  # pragma: no cover
         db.close()
 
 
+@contextlib.contextmanager
+def atomic_db():
+    # yield from get_db()
+    db = SessionLocal()
+    try:
+        yield db
+    finally:
+        db.close()
+
+
+sondes = {"test": schemas.SondeBase(identifiant="test", nom="Testlocal")}
+
+
+def apply_config(configs: list[dict] = []):
+    with atomic_db() as db:
+        for config in configs:
+            name = config["nom_sonde"].lower()
+            if "test" in name or "dummy" in name:
+                continue
+            identifiant = config["identifiant_sonde"]
+            sonde = crud.get_sonde(db, identifiant)
+            if sonde is None:
+                sonde = crud.create_sonde(db, identifiant, config["nom_sonde"])
+                logger.info(f"Create sonde {sonde.__dict__}")
+            else:
+                logger.info(f"{sonde.identifiant}: {sonde.nom} already exists")
+            PROBES[sonde.identifiant] = config
+
+
+apply_config(configurations)
+
+
 class Notifier:
     @staticmethod
     def __call__(idsonde, changes):
@@ -65,11 +116,32 @@ class Notifier:
                 "status": ["Le monitoring est inacessible"],
             }
         )
+        conf = PROBES[idsonde]
+
+        subject = "Probleme api supervision"
+        content = kind
+        from papi.mail_sendermodel import sendmail
+
+        if True:
+
+            sendmail(
+                htmlpart="",
+                emails=conf["emails"],
+                textpart=kind,
+                subject="probleme supervision api",
+            )
+
+    @staticmethod
+    def clean():
+        for k, v in notifications.items():
+            notifications[k] = v[-3:]
 
 
 Notifier = Notifier()
-
-sondes = {"test": schemas.SondeBase(identifiant="test", nom="Testlocal")}
+for k, sonde in PROBES.items():
+    sondes[k] = schemas.SondeBase(
+        identifiant=sonde["identifiant_sonde"], nom=sonde["nom_sonde"]
+    )
 
 
 def default_sample():
@@ -145,22 +217,29 @@ def post_sonde_error(
     db: Session = Depends(get_db),
 ):
     if not (sonde := crud.get_sonde(db, idsonde)):
-        return  # pragma: no cover
-    Notifier.error_sonde(idsonde)
+        if idsonde != "test":
+            return  # pragma: no cover
+
+    Notifier.error_sonde(idsonde, body["msg"])
+
     # create fake sample
-    last = list(crud.get_mesure(db, sonde.sonde_id, only_last=1))[0]
-    from copy import copy
+    try:
+        last = list(crud.get_mesure(db, sonde.sonde_id, only_last=1))[0]
+    except IndexError:
+        pass
+    else:
+        from copy import copy
 
-    api_error = copy(last.content)
-    from datetime import datetime
+        api_error = copy(last.content)
+        from datetime import datetime
 
-    date = str(datetime.now())
-    for channel in api_error["channels"]:
-        channel["status"] = "perte contact api"
+        date = str(datetime.now())
+        for channel in api_error["channels"]:
+            channel["status"] = "perte contact api"
 
-    api_error["date"] = date
+        api_error["date"] = date
 
-    post_sonde_data(request, idsonde, body=api_error, db=db)
+        post_sonde_data(request, idsonde, body=api_error, db=db)
     return
 
 
@@ -199,9 +278,15 @@ def post_sonde_data(
         html = res.body.decode("utf-8")
         from papi.mail_sendermodel import sendmail
 
-        cond = False 
-        if cond:
-            sendmail(html)
+        conf = PROBES[idsonde]
+        sumup = utils.sample2statuscount(present)
+        subject = " ; ".join(sumup)
+        sendmail(
+            htmlpart=html,
+            emails=conf["emails"],
+            subject=subject,
+        )
+
     return {
         "count": len(mesures_)
         if "date" in mesures_[0].content.keys()

papi/relais.py

@@ -0,0 +1,94 @@
+import requests
+import urllib3
+import sys
+from .config import read_config
+import logging as logger
+import json
+import math
+import itertools
+
+https = False
+
+
+urllib3.disable_warnings()
+
+
+session = requests.Session()
+session.verify = False
+
+logged = []
+
+
+def api_login(config):
+    post_url = config["api_login_url"]
+    login_r = session.post(
+        post_url,
+        json={"login": api_credentials.user, "password": api_credentials.password},
+    )
+    logged.append(True)
+
+    if login_r.ok:
+        logger.info("Logged")
+    else:  # pragma: no cover
+        logger.info("Login error")
+    return login_r
+
+
+def api_fetch(config):
+    if not logged:
+        api_login()
+    fetched = session.get("%s/status" % apiurl)
+    return fetched
+
+
+def api_forward(config):
+    post_url = ""
+    assert post_url
+    for post_url in forward_urls:
+        res = session.post(post_url, json=data)
+        print(res.ok)
+        print(res.json())
+
+
+def forward_api_error(message=""):
+    forward_urls = [
+        #        "https://papi.silib.re/sonde/test/error/",
+        "http://localhost:8000/sonde/838266b2-fc3a-4430-95e8-f7f0d0fc9871/error/",
+        #        "http://localhost:8000/sonde/test/error/",
+    ]
+    message = message or "Erreur inconnue"
+    for post_url in forward_urls:
+
+        res = session.post(post_url, json={"msg": message})
+        print(post_url)
+        print(res.ok)
+        print(res.json())
+
+    pass
+
+
+def main(
+    *,
+    maxloop=math.inf,
+    login=api_login,
+    fetch=api_fetch,
+    forward=api_forward,
+    forward_error=forward_api_error,
+    pattern="relais_*"
+):
+    loopcount = itertools.count().__next__
+    config = read_config(pattern=pattern)
+    assert len(config) == 1
+    config = config[0]
+    login(config)
+    while loopcount() < maxloop:
+        try:
+            current = fetch(config).json()
+        except Exception as E:
+            forward_error(str(E))
+        else:
+            forward(current)
+
+
+if __name__ == "__main__":
+    main(maxloop=1)

papi/sonde.py

@@ -1,260 +0,0 @@
-import requests
-import sys
-from time import sleep
-from copy import deepcopy
-
-try:
-    from . import api_credentials
-except ImportError:
-    import api_credentials
-import urllib3
-import math
-import itertools
-import datetime
-import csv
-import json
-import logging as logger
-import json
-
-https = False
-
-
-urllib3.disable_warnings()
-
-
-session = requests.Session()
-session.verify = False
-url = "https://%s" % api_credentials.ip
-
-
-apiurl = "%s/api/1.1" % url
-
-logged = []
-
-
-def login():
-    post_url = "%s/user/login" % apiurl
-    login_r = session.post(
-        post_url,
-        json={"login": api_credentials.user, "password": api_credentials.password},
-    )
-    logged.append(True)
-
-    if login_r.ok:
-        logger.info("Logged")
-    else:  # pragma: no cover
-        logger.info("Login error")
-    return login_r
-
-
-def forward(data):
-    forward_urls = [
-        "https://papi.silib.re/sonde/test/",
-    ]
-    for post_url in forward_urls:
-        res = session.post(post_url, json=data)
-        print(res.ok)
-        print(res.json())
-
-
-def status2list(status: dict):
-    keys = "id name channelType status".split(" ")
-    id_ = "id"
-    translate = {"channelType": "type"}
-
-    date = status["date"]
-    res = []
-    for key, channel in status["channels"].items():
-        res.append(
-            {"date": date, **{translate.get(key, key): channel[key] for key in keys}}
-        )
-    return sorted(res, key=lambda x: x[id_])
-
-
-def strip_channel(channel, keys=None):
-    """
-    >>> s = {'alarms': {'noSignal': True,
-    ...                 'noWatermark': None,
-    ...                 'qualityIndexStatus': None,
-    ...                 'timestampDrift': None,
-    ...                 'unexpectedWatermark': None},
-    ...      'channelType': 'fm',
-    ...      'id': 6,
-    ...      'lastTimestamp': None,
-    ...      'lastWatermarkId': None,
-    ...      'name': 'AES-67',
-    ...      'status': 'error'}
-
-    >>> strip_channel(s)
-    {'id': 6, 'name': 'AES-67', 'status': 'error'}
-
-    """
-    if keys is None:
-
-        keys = "id", "name", "status"
-
-    return {k: channel[k] for k in keys}
-
-
-def prepare(record):
-
-    newdict = deepcopy(record)
-    newdict["channels"] = dict(
-        sorted(channel2tuple(channel) for channel in record["channels"])
-    )
-    return newdict
-
-
-def fetch():
-    if not logged:
-        login()
-    fetched = session.get("%s/status" % apiurl)
-    return fetched
-
-
-def channel2tuple(channel):
-    return "%s#%s" % (channel["id"], channel["name"]), channel
-    pass
-
-
-def get_status(channels, key):
-    d = channels["channels"].get(key, {"status": "absent"})["status"]
-    return d
-
-
-def compare(channels, previous, current):
-    changes, states = [], []
-    for key in channels:
-        pstatus = get_status(previous, key)
-        cstatus = get_status(current, key)
-        state = key, pstatus, cstatus
-        if pstatus != cstatus:
-            changes.append(state)
-        states.append(state)
-
-    res = {}
-    if not changes:
-        return {}
-    disparus = [state for state in states if state[2] == "absent"]
-    if disparus:
-        res["disparus"] = disparus
-    apparus = [state for state in states if state[1] == "absent"]
-    if apparus:
-        res["apparus"] = apparus
-
-    res["changements"] = changes
-    res["etats"] = states
-    return res
-
-
-def load_or_fetch(fetch=fetch):
-    try:
-        with open("last.json") as f:
-            previous = json.load(f)
-        # print("found")
-    except (FileNotFoundError, json.decoder.JSONDecodeError):
-        print("Nolast")
-        with open("last.json", "w") as f:
-            previous = fetch().json()
-            json.dump(previous, f)
-    return previous
-
-
-def list_channels(p, c):
-    all_channels = sorted(set((*p["channels"], *c["channels"])))
-    return all_channels
-
-
-def main(*, maxloop=math.inf, login=login, fetch=fetch):
-    loopcount = itertools.count().__next__
-
-    login()
-    previous = load_or_fetch()
-
-    historique = []
-    while loopcount() < maxloop:
-        try:
-
-            current = fetch().json()
-        except json.decoder.JSONDecodeError:
-            breakpoint()
-        forward(current)
-        """"
-        with open("last.json", "w") as f:
-            json.dump(current, f)
-
-        with open(raw_filename(), "a") as f:
-            json.dump(current, f)
-
-        current, previous = prepare(current), prepare(previous)
-        all_channels = sorted(set((*previous["channels"], *current["channels"])))
-
-        savelog2csv(current)
-
-        diff = compare(all_channels, previous, current)
-
-        savediff(date=current["date"], diff=diff)
-
-        if diff:
-            print("**********")
-            print(diff["changements"])
-            print("!!!!!!!!!!")
-            historique.append(diff)
-            previous = current
-        sleep(0.5)
-    return historique
-        """
-
-
-def make_id_key(channel, keys=None, sep="#", tuple_=False):
-    """
-    This takes out the concatenation of keys, value to use is as a pair.
-
-    >>> sample = {'id': 6, 'name': 'foo'}
-    >>> make_id_key(sample)
-    {'6#foo': {'id': 6, 'name': 'foo'}}
-    """
-    if not keys:
-        keys = ["id", "name"]
-    kvalue = sep.join(str(channel[k]) for k in keys)
-    if tuple_:
-        return kvalue, channel
-    return {kvalue: channel}
-
-
-def raw_filename():
-    return "raw_" + str(datetime.date.today()).replace("-", "_") + ".json"
-
-
-def log_filename():
-    return "log_" + str(datetime.date.today()).replace("-", "_") + ".csv"
-
-
-def diff_filename():
-    return "diff_" + str(datetime.date.today()).replace("-", "_") + ".csv"
-
-
-def savelog2csv(alert, *, filename_f=log_filename):
-    keys = "date id name type status".split(" ")
-    with open(filename_f(), "a") as f:
-        writer = csv.DictWriter(f, keys)
-        if f.tell() == 0:
-            writer.writeheader()
-        for a in status2list(alert):
-            writer.writerow(a)
-
-
-def savediff(date, diff, *, filename=diff_filename):
-    keys = "date name before after".split(" ")
-    with open(filename(), "a") as f:
-        writer = csv.DictWriter(f, keys)
-        if f.tell() == 0:
-            writer.writeheader()
-        for d in diff:
-            data = {"date": date}
-            data.update(zip(("name", "before", "after"), d))
-            writer.writerow(data)
-
-
-if __name__ == "__main__":
-    main(maxloop=1)

papi/utils.py

@@ -5,6 +5,7 @@ from copy import deepcopy
 
 import math
 import itertools
+import collections
 import datetime
 import csv
 import json
@@ -29,6 +30,16 @@ def get_status(channels, key):
     return d
 
 
+def sample2statuscount(prepared):
+    sumup = [
+        "%s: %d" % (k, v)
+        for k, v in collections.Counter(
+            x["status"] for x in prepared["channels"].values()
+        ).items()
+    ]
+    return sumup
+
+
 def compare(channels, previous, current):
     changes, states = [], []
     for key in channels:

tests/test_papi.py

@@ -8,6 +8,9 @@ from fastapi.testclient import TestClient
 from papi.main import app, get_db
 
 from papi.main import sondes
+import papi.main
+from papi import main
+
 
 from papi import __version__
 from . import utils as testutils