First Commit

epg2xml/__init__.py

@@ -0,0 +1,15 @@
+__title__ = "epg2xml"
+__description__ = "웹 상의 소스를 취합하여 EPG를 만드는 프로그램"
+__url__ = "https://github.com/epg2xml/epg2xml"
+
+try:
+    from ._version import version
+except ImportError:
+    try:
+        from setuptools_scm import get_version
+
+        version = get_version(version_scheme="release-branch-semver")
+    except Exception:
+        version = "2.6.0.dev0"
+
+__version__ = version

epg2xml/__main__.py

@@ -0,0 +1,72 @@
+import logging
+import socket
+import sys
+from contextlib import ExitStack
+
+from epg2xml.config import Config
+from epg2xml.providers import EPGHandler
+
+############################################################
+# INIT
+############################################################
+
+# load initial config
+conf = Config()
+
+# load config file
+conf.load()
+
+# logger
+log = logging.getLogger("MAIN")
+
+############################################################
+# MAIN
+############################################################
+
+
+def main():
+    log.debug("Loading providers...")
+    h = EPGHandler(conf.configs)
+
+    if (cmd := conf.args["cmd"]) in ["run", "fromdb"]:
+        with ExitStack() as stack:
+            # redirecting stdout to...
+            if xmlfile := conf.settings["xmlfile"]:
+                sys.stdout = stack.enter_context(open(xmlfile, "w", encoding="utf-8"))
+            elif xmlsock := conf.settings["xmlsock"]:
+                sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+                sock.connect(xmlsock)
+                sys.stdout = stack.enter_context(sock.makefile("w"))
+
+            if cmd == "fromdb":
+                log.debug("Importing from dbfile...")
+                h.from_db(conf.settings["dbfile"])
+            else:
+                log.debug("Loading service channels...")
+                h.load_channels(conf.settings["channelfile"], conf.settings["parallel"])
+
+                log.debug("Loading requested channels...")
+                h.load_req_channels()
+
+                log.debug("Getting EPG...")
+                h.get_programs(conf.settings["parallel"])
+
+                if (dbfile := conf.settings["dbfile"]) is not None:
+                    log.debug("Exporting to dbfile...")
+                    h.to_db(dbfile)
+
+            log.info("Writing xmltv.dtd header...")
+            h.to_xml()
+
+            log.info("Done")
+    elif cmd == "update_channels":
+        h.load_channels(conf.settings["channelfile"], conf.settings["parallel"])
+    else:
+        raise NotImplementedError(f"Unknown command: {cmd}")
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except KeyboardInterrupt:
+        sys.exit(0)

epg2xml/config.py

@@ -0,0 +1,389 @@
+import argparse
+import errno
+import json
+import logging
+import os
+import sys
+from copy import copy
+from logging.handlers import RotatingFileHandler
+from pathlib import Path
+from typing import Union
+
+from epg2xml import __description__, __title__, __url__, __version__
+from epg2xml.utils import dump_json
+
+# suppress modules logging
+logging.getLogger("requests").setLevel(logging.ERROR)
+logging.getLogger("urllib3.connectionpool").setLevel(logging.ERROR)
+
+logger = logging.getLogger("CONFIG")
+
+
+def setup_root_logger(
+    *,
+    handler: logging.Handler = None,
+    formatter: logging.Formatter = None,
+    level: Union[int, str] = None,
+) -> None:
+    if level is None:
+        level = logging.INFO
+
+    if handler is None:
+        # logging to console, stderr by default
+        handler = logging.StreamHandler()
+
+    if formatter is None:
+        log_fmt = "%(asctime)-15s %(levelname)-8s %(name)-7s %(lineno)4d: %(message)s"
+        formatter = logging.Formatter(log_fmt, datefmt="%Y/%m/%d %H:%M:%S")
+
+    handler.setFormatter(formatter)
+    logging.getLogger().addHandler(handler)
+    logging.getLogger().setLevel(level)
+
+
+class Singleton(type):
+    _instances = {}
+
+    def __call__(cls, *args, **kwargs):
+        if cls not in cls._instances:
+            cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs)
+
+        return cls._instances[cls]
+
+
+class Config:
+    __metaclass__ = Singleton
+
+    base_config = {
+        "GLOBAL": {
+            "ENABLED": True,
+            "FETCH_LIMIT": 2,
+            "ID_FORMAT": "{ServiceId}.{Source.lower()}",
+            "ADD_REBROADCAST_TO_TITLE": False,
+            "ADD_EPNUM_TO_TITLE": True,
+            "ADD_DESCRIPTION": True,
+            "ADD_XMLTV_NS": False,
+            "GET_MORE_DETAILS": False,
+            "ADD_CHANNEL_ICON": True,
+            "HTTP_PROXY": None,
+        },
+        "KT": {
+            "MY_CHANNELS": [],
+        },
+        "LG": {
+            "MY_CHANNELS": [],
+        },
+        "SK": {
+            "MY_CHANNELS": [],
+        },
+        "DAUM": {
+            "MY_CHANNELS": [],
+        },
+        "NAVER": {
+            "MY_CHANNELS": [],
+        },
+        "WAVVE": {
+            "MY_CHANNELS": [],
+        },
+        "TVING": {
+            "MY_CHANNELS": [],
+        },
+        "SPOTV": {
+            "MY_CHANNELS": [],
+        },
+    }
+
+    base_settings = {
+        "config": {
+            "argv": "--config",
+            "env": "EPG2XML_CONFIG",
+            "default": str(Path.cwd().joinpath("epg2xml.json")),
+        },
+        "logfile": {
+            "argv": "--logfile",
+            "env": "EPG2XML_LOGFILE",
+            "default": None,
+        },
+        "loglevel": {
+            "argv": "--loglevel",
+            "env": "EPG2XML_LOGLEVEL",
+            "default": "INFO",
+        },
+        "channelfile": {
+            "argv": "--channelfile",
+            "env": "EPG2XML_CHANNELFILE",
+            "default": str(Path.cwd().joinpath("Channel.json")),
+        },
+        "xmlfile": {
+            "argv": "--xmlfile",
+            "env": "EPG2XML_XMLFILE",
+            "default": None,
+        },
+        "xmlsock": {
+            "argv": "--xmlsock",
+            "env": "EPG2XML_XMLSOCK",
+            "default": None,
+        },
+        "parallel": {
+            "argv": "--parallel",
+            "env": "EPG2XML_PARALLEL",
+            "default": False,
+        },
+        "dbfile": {
+            "argv": "--dbfile",
+            "env": "EPG2XML_DBFILE",
+            "default": None,
+        },
+    }
+
+    def __init__(self):
+        """Initializes config"""
+        # Args and settings
+        self.args = self.parse_args()
+        self.settings = self.get_settings()
+        # Configs
+        self.configs = None
+
+    @property
+    def default_config(self):
+        """reserved for adding extra fields"""
+        cfg = copy(self.base_config)
+        return cfg
+
+    def __inner_upgrade(self, settings1, settings2, key=None, overwrite=False):
+        sub_upgraded = False
+        merged = copy(settings2)
+
+        if isinstance(settings1, dict):
+            for k, v in settings1.items():
+                # missing k
+                if k not in settings2:
+                    merged[k] = v
+                    sub_upgraded = True
+                    if not key:
+                        logger.info("Added %r config option: %s", str(k), str(v))
+                    else:
+                        logger.info("Added %r to config option %r: %s", str(k), str(key), str(v))
+                    continue
+
+                # iterate children
+                if isinstance(v, (dict, list)):
+                    merged[k], did_upgrade = self.__inner_upgrade(
+                        settings1[k], settings2[k], key=k, overwrite=overwrite
+                    )
+                    sub_upgraded = did_upgrade or sub_upgraded
+                elif settings1[k] != settings2[k] and overwrite:
+                    merged = settings1
+                    sub_upgraded = True
+        elif isinstance(settings1, list) and key:
+            for v in settings1:
+                if v not in settings2:
+                    merged.append(v)
+                    sub_upgraded = True
+                    logger.info("Added to config option %r: %s", str(key), str(v))
+                    continue
+
+        return merged, sub_upgraded
+
+    def upgrade_configs(self, currents):
+        fields_env = {}
+
+        # ENV gets priority: ENV > config.json
+        for name, _ in self.base_config.items():
+            if name in os.environ:
+                # Use JSON decoder to get same behaviour as config file
+                fields_env[name] = json.JSONDecoder().decode(os.environ[name])
+                logger.debug("setting from ENV   --%s=%s", name, fields_env[name])
+
+        # Update in-memory config with environment settings
+        currents.update(fields_env)
+
+        # Do inner upgrade
+        upgraded_configs, upgraded = self.__inner_upgrade(self.base_config, currents)
+        return upgraded_configs, upgraded
+
+    def load_with_hidden(self, cfg_old):
+        cfg_new = copy(cfg_old)
+        for p in cfg_new:
+            # push items in GLOBAL as defaults
+            for k, v in cfg_old["GLOBAL"].items():
+                if k not in cfg_new[p]:
+                    cfg_new[p][k] = v
+        del cfg_new["GLOBAL"]
+        self.configs = cfg_new
+
+    def load(self):
+        logger.debug("Loading config...")
+        if not Path(self.settings["config"]).exists():
+            logger.info("No config file found. Creating a default one...")
+            self.save(self.default_config)
+
+        try:
+            with open(self.settings["config"], "r", encoding="utf-8") as fp:
+                cfg, upgraded = self.upgrade_configs(json.load(fp))
+
+                # Save config if upgraded
+                if upgraded:
+                    self.save(cfg)
+                    sys.exit(0)
+
+            self.load_with_hidden(cfg)
+        except (json.decoder.JSONDecodeError, ValueError):
+            logger.exception("Please check your config here: %s", self.settings["config"])
+            sys.exit(1)
+
+    def save(self, cfg, exitOnSave=True):
+        dump_json(self.settings["config"], cfg)
+        if exitOnSave:
+            logger.info("Your config was upgraded. You may check the changes here: %r", self.settings["config"])
+
+        if exitOnSave:
+            sys.exit(0)
+
+    def get_settings(self):
+        setts = {}
+        for name, data in self.base_settings.items():
+            # Argrument priority: cmd < environment < default
+            try:
+                value = None
+                # Command line argument
+                if self.args[name]:
+                    value = self.args[name]
+                    logger.debug("setting from ARG   --%s=%s", name, value)
+
+                # Envirnoment variable
+                elif data["env"] in os.environ:
+                    value = os.environ[data["env"]]
+                    logger.debug("setting from ENV   --%s=%s", data["env"], value)
+
+                # Default
+                else:
+                    value = data["default"]
+                    logger.debug("setting by default %s=%s", data["argv"], value)
+
+                setts[name] = value
+
+            except Exception:
+                logger.exception("Exception raised on setting value: %r", name)
+
+        # checking existance of important files' dir
+        for argname in ["config", "logfile", "channelfile", "dbfile"]:
+            filepath = setts[argname]
+            if filepath is not None and not Path(filepath).parent.exists():
+                logger.error(FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), filepath))
+                sys.exit(1)
+
+        # handling of boolean args
+        for argname in ["parallel"]:
+            if isinstance(setts[argname], str):
+                setts[argname] = setts[argname].lower() in ("y", "yes", "t", "true", "on", "1")
+
+        # logging to file
+        if setts["logfile"] is not None:
+            fileHandler = RotatingFileHandler(setts["logfile"], maxBytes=2 * 1024**2, backupCount=5, encoding="utf-8")
+            setup_root_logger(handler=fileHandler)
+
+        # set configured log level
+        logging.getLogger().setLevel(setts["loglevel"])
+
+        return setts
+
+    # Parse command line arguments
+    def parse_args(self):
+        parser = argparse.ArgumentParser(
+            prog=__title__,
+            description=__description__,
+            epilog=f"Online help: <{__url__}>",
+            formatter_class=argparse.RawTextHelpFormatter,
+        )
+
+        # Mode
+        parser.add_argument(
+            "cmd",
+            metavar="command",
+            choices=("run", "fromdb", "update_channels"),
+            help=('"run": XML 형식으로 출력\n' '"fromdb": dbfile로부터 불러오기\n' '"update_channels": 채널 정보 업데이트'),
+        )
+
+        # Display version info
+        parser.add_argument(
+            "-v",
+            "--version",
+            action="version",
+            version=f"{__title__} v{__version__}",
+        )
+
+        # Config file
+        parser.add_argument(
+            self.base_settings["config"]["argv"],
+            nargs="?",
+            const=None,
+            help=f"config file path (default: {self.base_settings['config']['default']})",
+        )
+
+        # Log file
+        parser.add_argument(
+            self.base_settings["logfile"]["argv"],
+            nargs="?",
+            const=None,
+            help=f"log file path (default: {self.base_settings['logfile']['default']})",
+        )
+
+        # Log level
+        parser.add_argument(
+            self.base_settings["loglevel"]["argv"],
+            choices=("DEBUG", "INFO", "WARNING", "ERROR"),
+            help=f"loglevel (default: {self.base_settings['loglevel']['default']})",
+        )
+
+        # Channel file
+        parser.add_argument(
+            self.base_settings["channelfile"]["argv"],
+            nargs="?",
+            const=None,
+            help=f"channel file path (default: {self.base_settings['channelfile']['default']})",
+        )
+
+        # XML file
+        parser.add_argument(
+            self.base_settings["xmlfile"]["argv"],
+            nargs="?",
+            const=None,
+            help="write output to file if specified",
+        )
+
+        # XML socket
+        parser.add_argument(
+            self.base_settings["xmlsock"]["argv"],
+            nargs="?",
+            const=None,
+            help="send output to unix socket if specified",
+        )
+
+        # Run in Parallel
+        parser.add_argument(
+            self.base_settings["parallel"]["argv"],
+            action="store_true",
+            help="run in parallel",
+        )
+
+        # DB file
+        parser.add_argument(
+            self.base_settings["dbfile"]["argv"],
+            nargs="?",
+            const=None,
+            help="export/import data to/from db",
+        )
+
+        # Print help by default if no arguments
+        if len(sys.argv) == 1:
+            parser.print_help()
+
+            sys.exit(0)
+
+        else:
+            return vars(parser.parse_args())
+
+
+# logging
+setup_root_logger()

epg2xml/epg2xml.json

@@ -0,0 +1,299 @@
+{
+  "GLOBAL": {
+    "ENABLED": true,
+    "FETCH_LIMIT": 2,
+    "ID_FORMAT": "{ServiceId}.{Source.lower()}",
+    "ADD_REBROADCAST_TO_TITLE": false,
+    "ADD_EPNUM_TO_TITLE": true,
+    "ADD_DESCRIPTION": true,
+    "ADD_XMLTV_NS": false,
+    "GET_MORE_DETAILS": false,
+    "ADD_CHANNEL_ICON": true,
+    "HTTP_PROXY": null
+  },
+  "KT": {
+    "MY_CHANNELS": [
+	  { "Name": "SBS", "No": "5", "ServiceId": "5" },
+      { "Name": "KBS2", "No": "7", "ServiceId": "7" },
+      { "Name": "KBS1", "No": "9", "ServiceId": "9" },
+      { "Name": "MBC", "No": "11", "ServiceId": "11" },
+      { "Name": "EBS", "No": "13", "ServiceId": "13" },
+      { "Name": "EBS2", "No": "95", "ServiceId": "95" },
+      { "Name": "NS홈쇼핑", "No": "2", "ServiceId": "2" },
+      { "Name": "롯데홈쇼핑", "No": "4", "ServiceId": "4" },
+      { "Name": "CJ ONSTYLE", "No": "6", "ServiceId": "6" },
+      { "Name": "GS SHOP", "No": "8", "ServiceId": "8" },
+      { "Name": "현대홈쇼핑", "No": "10", "ServiceId": "10" },
+      { "Name": "kt알파쇼핑", "No": "12", "ServiceId": "12" },
+      { "Name": "홈&쇼핑", "No": "14", "ServiceId": "14" },
+      { "Name": "OBS", "No": "26", "ServiceId": "26" },
+      { "Name": "SK stoa", "No": "17", "ServiceId": "17" },
+      { "Name": "신세계쇼핑", "No": "20", "ServiceId": "20" },
+      { "Name": "공영쇼핑", "No": "22", "ServiceId": "22" },
+      { "Name": "GS MY SHOP", "No": "28", "ServiceId": "28" },
+      { "Name": "쇼핑엔티", "No": "30", "ServiceId": "30" },
+      { "Name": "LOTTE OneTV", "No": "32", "ServiceId": "32" },
+      { "Name": "W쇼핑", "No": "34", "ServiceId": "34" },
+      { "Name": "현대홈쇼핑+샵", "No": "36", "ServiceId": "36" },
+      { "Name": "CJ ONSTYLE+", "No": "38", "ServiceId": "38" },
+      { "Name": "NS Shop+", "No": "42", "ServiceId": "42" },
+      { "Name": "JTBC", "No": "15", "ServiceId": "15" },
+      { "Name": "MBN", "No": "16", "ServiceId": "16" },
+      { "Name": "채널A", "No": "18", "ServiceId": "18" },
+      { "Name": "TV조선", "No": "19", "ServiceId": "19" },
+      { "Name": "ENA", "No": "1", "ServiceId": "1" },
+      { "Name": "tvN", "No": "3", "ServiceId": "3" },
+      { "Name": "tvN STORY", "No": "21", "ServiceId": "21" },
+      { "Name": "ENA DRAMA", "No": "29", "ServiceId": "29" },
+      { "Name": "MBC Dramanet", "No": "31", "ServiceId": "31" },
+      { "Name": "KBS Drama", "No": "35", "ServiceId": "35" },
+      { "Name": "SBS Plus", "No": "37", "ServiceId": "37" },
+      { "Name": "JTBC2", "No": "39", "ServiceId": "39" },
+      { "Name": "tvN DRAMA", "No": "45", "ServiceId": "45" },
+      { "Name": "드라마큐브", "No": "46", "ServiceId": "46" },
+      { "Name": "Dramax", "No": "47", "ServiceId": "47" },
+      { "Name": "E채널", "No": "48", "ServiceId": "48" },
+      { "Name": "GTV", "No": "67", "ServiceId": "67" },
+      { "Name": "CNTV", "No": "68", "ServiceId": "68" },
+      { "Name": "TVCHOSUN2", "No": "69", "ServiceId": "69" },
+      { "Name": "FUN TV", "No": "71", "ServiceId": "71" },
+      { "Name": "하이라이트TV", "No": "74", "ServiceId": "74" },
+      { "Name": "JTBC4", "No": "75", "ServiceId": "75" },
+      { "Name": "Lifetime", "No": "78", "ServiceId": "78" },
+      { "Name": "Edge TV", "No": "79", "ServiceId": "79" },
+      { "Name": "MBC ON", "No": "80", "ServiceId": "80" },
+      { "Name": "동아TV", "No": "82", "ServiceId": "82" },
+      { "Name": "KBS Story", "No": "83", "ServiceId": "83" },
+      { "Name": "ONCE", "No": "88", "ServiceId": "88" },
+      { "Name": "디원", "No": "89", "ServiceId": "89" },
+      { "Name": "WeLike", "No": "146", "ServiceId": "146" },
+      { "Name": "MBC Every1", "No": "0", "ServiceId": "0" },
+      { "Name": "Mnet", "No": "27", "ServiceId": "27" },
+      { "Name": "tvN SHOW", "No": "40", "ServiceId": "40" },
+      { "Name": "KBS Joy", "No": "41", "ServiceId": "41" },
+      { "Name": "SBS funE", "No": "43", "ServiceId": "43" },
+      { "Name": "채널S", "No": "44", "ServiceId": "44" },
+      { "Name": "디스커버리채널", "No": "50", "ServiceId": "50" },
+      { "Name": "SBS F!L", "No": "66", "ServiceId": "66" },
+      { "Name": "ENA STORY", "No": "72", "ServiceId": "72" },
+      { "Name": "스마일티브이플러스", "No": "84", "ServiceId": "84" },
+      { "Name": "코미디TV", "No": "85", "ServiceId": "85" },
+      { "Name": "K STAR", "No": "87", "ServiceId": "87" },
+      { "Name": "아이넷TV", "No": "92", "ServiceId": "92" },
+      { "Name": "채널이엠", "No": "93", "ServiceId": "93" },
+      { "Name": "CMCTV", "No": "94", "ServiceId": "94" },
+      { "Name": "엔터TV", "No": "96", "ServiceId": "96" },
+      { "Name": "채널A 플러스", "No": "98", "ServiceId": "98" },
+      { "Name": "MBN Plus", "No": "99", "ServiceId": "99" },
+      { "Name": "HQ+", "No": "111", "ServiceId": "111" },
+      { "Name": "CH.WIDE", "No": "130", "ServiceId": "130" },
+      { "Name": "SPOTV K", "No": "135", "ServiceId": "135" },
+      { "Name": "SBS M", "No": "136", "ServiceId": "136" },
+      { "Name": "MBC M", "No": "137", "ServiceId": "137" },
+      { "Name": "뉴트로TV", "No": "138", "ServiceId": "138" },
+      { "Name": "ORFEO", "No": "139", "ServiceId": "139" },
+      { "Name": "한경arteTV", "No": "140", "ServiceId": "140" },
+      { "Name": "History", "No": "141", "ServiceId": "141" },
+      { "Name": "GMTV", "No": "142", "ServiceId": "142" },
+      { "Name": "가요TV", "No": "143", "ServiceId": "143" },
+      { "Name": "실버아이TV", "No": "144", "ServiceId": "144" },
+      { "Name": "이벤트TV", "No": "145", "ServiceId": "145" },
+      { "Name": "붐TV", "No": "147", "ServiceId": "147" },
+      { "Name": "아이넷라이프", "No": "148", "ServiceId": "148" },
+      { "Name": "ENA PLAY", "No": "999", "ServiceId": "999" },
+      { "Name": "OCN", "No": "33", "ServiceId": "33" },
+      { "Name": "시네마천국", "No": "49", "ServiceId": "49" },
+      { "Name": "AsiaN", "No": "73", "ServiceId": "73" },
+      { "Name": "OCN Movies2", "No": "76", "ServiceId": "76" },
+      { "Name": "AsiaM", "No": "90", "ServiceId": "90" },
+      { "Name": "월드 클래식 무비", "No": "91", "ServiceId": "91" },
+      { "Name": "스크린", "No": "101", "ServiceId": "101" },
+      { "Name": "채널차이나", "No": "102", "ServiceId": "102" },
+      { "Name": "mplex", "No": "103", "ServiceId": "103" },
+      { "Name": "THE MOVIE", "No": "104", "ServiceId": "104" },
+      { "Name": "인디필름", "No": "105", "ServiceId": "105" },
+      { "Name": "씨네프", "No": "106", "ServiceId": "106" },
+      { "Name": "채널나우", "No": "107", "ServiceId": "107" },
+      { "Name": "채널 J", "No": "108", "ServiceId": "108" },
+      { "Name": "에이플드라마", "No": "109", "ServiceId": "109" },
+      { "Name": "중화TV", "No": "110", "ServiceId": "110" },
+      { "Name": "CH.U", "No": "112", "ServiceId": "112" },
+      { "Name": "AXN", "No": "113", "ServiceId": "113" },
+      { "Name": "텔레노벨라", "No": "114", "ServiceId": "114" },
+      { "Name": "Focus Prime", "No": "115", "ServiceId": "115" },
+      { "Name": "채널W", "No": "116", "ServiceId": "116" },
+      { "Name": "TVasiaPlus", "No": "117", "ServiceId": "117" },
+      { "Name": "HITS", "No": "118", "ServiceId": "118" },
+      { "Name": "OCN Movies", "No": "998", "ServiceId": "998" },
+      { "Name": "SPOTV", "No": "51", "ServiceId": "51" },
+      { "Name": "SPOTV2", "No": "52", "ServiceId": "52" },
+      { "Name": "IB SPORTS", "No": "53", "ServiceId": "53" },
+      { "Name": "SkySports", "No": "54", "ServiceId": "54" },
+      { "Name": "GOLF&PBA", "No": "55", "ServiceId": "55" },
+      { "Name": "JTBC Golf", "No": "56", "ServiceId": "56" },
+      { "Name": "SBSGOLF", "No": "57", "ServiceId": "57" },
+      { "Name": "SBS Sports", "No": "58", "ServiceId": "58" },
+      { "Name": "KBS N Sports", "No": "59", "ServiceId": "59" },
+      { "Name": "MBC SPORTS+", "No": "60", "ServiceId": "60" },
+      { "Name": "JTBC GOLF&SPORTS", "No": "61", "ServiceId": "61" },
+      { "Name": "SBS Golf2", "No": "62", "ServiceId": "62" },
+      { "Name": "SPOTV Golf & Health", "No": "63", "ServiceId": "63" },
+      { "Name": "tvN SPORTS", "No": "77", "ServiceId": "77" },
+      { "Name": "Eurosport", "No": "119", "ServiceId": "119" },
+      { "Name": "OLIFE", "No": "86", "ServiceId": "86" },
+      { "Name": "FTV", "No": "120", "ServiceId": "120" },
+      { "Name": "한국낚시방송", "No": "121", "ServiceId": "121" },
+      { "Name": "바둑TV", "No": "122", "ServiceId": "122" },
+      { "Name": "K바둑", "No": "123", "ServiceId": "123" },
+      { "Name": "브레인TV", "No": "126", "ServiceId": "126" },
+      { "Name": "빌리어즈티비", "No": "127", "ServiceId": "127" },
+      { "Name": "마운틴TV", "No": "128", "ServiceId": "128" },
+      { "Name": "AfreecaTV", "No": "129", "ServiceId": "129" },
+      { "Name": "STN", "No": "131", "ServiceId": "131" },
+      { "Name": "생활체육TV", "No": "132", "ServiceId": "132" },
+      { "Name": "스크린골프존", "No": "133", "ServiceId": "133" },
+      { "Name": "STORYTV", "No": "134", "ServiceId": "134" },
+      { "Name": "MAXPORTS", "No": "167", "ServiceId": "167" },
+      { "Name": "폴라리스TV", "No": "226", "ServiceId": "226" },
+      { "Name": "채널 키즈랜드", "No": "960", "ServiceId": "960" },
+      { "Name": "ZooMoo", "No": "961", "ServiceId": "961" },
+      { "Name": "Dream Works Channel", "No": "962", "ServiceId": "962" },
+      { "Name": "Miao Mi", "No": "968", "ServiceId": "968" },
+      { "Name": "뽀요TV", "No": "976", "ServiceId": "976" },
+      { "Name": "Cbeebies", "No": "977", "ServiceId": "977" },
+      { "Name": "브라보키즈", "No": "980", "ServiceId": "980" },
+      { "Name": "대교 노리Q", "No": "981", "ServiceId": "981" },
+      { "Name": "EBS KIDS", "No": "983", "ServiceId": "983" },
+      { "Name": "KBS Kids", "No": "984", "ServiceId": "984" },
+      { "Name": "캐리TV", "No": "985", "ServiceId": "985" },
+      { "Name": "JEI 재능TV", "No": "986", "ServiceId": "986" },
+      { "Name": "대교 어린이TV", "No": "987", "ServiceId": "987" },
+      { "Name": "핑크퐁채널", "No": "988", "ServiceId": "988" },
+      { "Name": "부메랑", "No": "989", "ServiceId": "989" },
+      { "Name": "애니플러스", "No": "990", "ServiceId": "990" },
+      { "Name": "카툰네트워크", "No": "991", "ServiceId": "991" },
+      { "Name": "애니박스", "No": "993", "ServiceId": "993" },
+      { "Name": "애니원", "No": "994", "ServiceId": "994" },
+      { "Name": "ANIMAX", "No": "995", "ServiceId": "995" },
+      { "Name": "Tooniverse", "No": "996", "ServiceId": "996" },
+      { "Name": "다빈치러닝", "No": "969", "ServiceId": "969" },
+      { "Name": "edu TV", "No": "970", "ServiceId": "970" },
+      { "Name": "EBS PLUS2", "No": "971", "ServiceId": "971" },
+      { "Name": "EBS PLUS1", "No": "972", "ServiceId": "972" },
+      { "Name": "EBS English", "No": "973", "ServiceId": "973" },
+      { "Name": "플레이런TV", "No": "974", "ServiceId": "974" },
+      { "Name": "JEI EnglishTV", "No": "975", "ServiceId": "975" },
+      { "Name": "NBS한국농업방송", "No": "100", "ServiceId": "100" },
+      { "Name": "엑스원", "No": "156", "ServiceId": "156" },
+      { "Name": "KBS LIFE", "No": "158", "ServiceId": "158" },
+      { "Name": "리얼TV", "No": "161", "ServiceId": "161" },
+      { "Name": "Now제주TV", "No": "162", "ServiceId": "162" },
+      { "Name": "9colors", "No": "163", "ServiceId": "163" },
+      { "Name": "MBC NET", "No": "164", "ServiceId": "164" },
+      { "Name": "BBC Earth", "No": "172", "ServiceId": "172" },
+      { "Name": "HGTV", "No": "173", "ServiceId": "173" },
+      { "Name": "Animal Planet", "No": "174", "ServiceId": "174" },
+      { "Name": "YTN 사이언스", "No": "175", "ServiceId": "175" },
+      { "Name": "채널뷰", "No": "176", "ServiceId": "176" },
+      { "Name": "CCTV4", "No": "177", "ServiceId": "177" },
+      { "Name": "Discovery Science", "No": "178", "ServiceId": "178" },
+      { "Name": "연합뉴스TV", "No": "23", "ServiceId": "23" },
+      { "Name": "YTN", "No": "24", "ServiceId": "24" },
+      { "Name": "SBS Biz", "No": "25", "ServiceId": "25" },
+      { "Name": "한국경제TV", "No": "180", "ServiceId": "180" },
+      { "Name": "MTN 머니투데이방송", "No": "181", "ServiceId": "181" },
+      { "Name": "매일경제TV", "No": "182", "ServiceId": "182" },
+      { "Name": "이데일리TV", "No": "183", "ServiceId": "183" },
+      { "Name": "서울경제TV", "No": "184", "ServiceId": "184" },
+      { "Name": "토마토증권통", "No": "185", "ServiceId": "185" },
+      { "Name": "팍스경제TV", "No": "186", "ServiceId": "186" },
+      { "Name": "연합뉴스경제TV", "No": "187", "ServiceId": "187" },
+      { "Name": "토마토집통", "No": "188", "ServiceId": "188" },
+      { "Name": "NHK WP", "No": "189", "ServiceId": "189" },
+      { "Name": "ABC Australia", "No": "190", "ServiceId": "190" },
+      { "Name": "CNN International", "No": "191", "ServiceId": "191" },
+      { "Name": "BBC News", "No": "192", "ServiceId": "192" },
+      { "Name": "Euro News", "No": "193", "ServiceId": "193" },
+      { "Name": "CGTN", "No": "194", "ServiceId": "194" },
+      { "Name": "Fox News", "No": "195", "ServiceId": "195" },
+      { "Name": "Bloomberg", "No": "196", "ServiceId": "196" },
+      { "Name": "CNBC", "No": "197", "ServiceId": "197" },
+      { "Name": "TV5MONDE", "No": "198", "ServiceId": "198" },
+      { "Name": "DW-TV Asia+", "No": "200", "ServiceId": "200" },
+      { "Name": "KTV", "No": "64", "ServiceId": "64" },
+      { "Name": "국회방송", "No": "65", "ServiceId": "65" },
+      { "Name": "HD OBS W", "No": "81", "ServiceId": "81" },
+      { "Name": "다문화티브이", "No": "97", "ServiceId": "97" },
+      { "Name": "컬쳐플러스", "No": "149", "ServiceId": "149" },
+      { "Name": "MGTV", "No": "157", "ServiceId": "157" },
+      { "Name": "YTN2", "No": "159", "ServiceId": "159" },
+      { "Name": "OUN", "No": "160", "ServiceId": "160" },
+      { "Name": "채널i", "No": "165", "ServiceId": "165" },
+      { "Name": "아리랑 TV", "No": "166", "ServiceId": "166" },
+      { "Name": "쿠키건강TV", "No": "169", "ServiceId": "169" },
+      { "Name": "메디컬TV", "No": "171", "ServiceId": "171" },
+      { "Name": "복지TV", "No": "199", "ServiceId": "199" },
+      { "Name": "법률방송", "No": "213", "ServiceId": "213" },
+      { "Name": "TBS TV", "No": "214", "ServiceId": "214" },
+      { "Name": "헬스메디tv", "No": "215", "ServiceId": "215" },
+      { "Name": "육아방송", "No": "217", "ServiceId": "217" },
+      { "Name": "K-NET TV", "No": "221", "ServiceId": "221" },
+      { "Name": "시니어TV", "No": "222", "ServiceId": "222" },
+      { "Name": "소상공인시장tv", "No": "223", "ServiceId": "223" },
+      { "Name": "지방자치TV", "No": "224", "ServiceId": "224" },
+      { "Name": "디마티비", "No": "225", "ServiceId": "225" },
+      { "Name": "가톨릭평화방송", "No": "231", "ServiceId": "231" },
+      { "Name": "BBS불교방송", "No": "232", "ServiceId": "232" },
+      { "Name": "BTN불교TV", "No": "233", "ServiceId": "233" },
+      { "Name": "Good TV", "No": "234", "ServiceId": "234" },
+      { "Name": "C Channel", "No": "235", "ServiceId": "235" },
+      { "Name": "CTS기독교TV", "No": "236", "ServiceId": "236" },
+      { "Name": "CGN", "No": "237", "ServiceId": "237" },
+      { "Name": "CBS", "No": "238", "ServiceId": "238" },
+      { "Name": "원음방송", "No": "239", "ServiceId": "239" },
+      { "Name": "YCN유림방송", "No": "240", "ServiceId": "240" },
+      { "Name": "STB상생방송", "No": "241", "ServiceId": "241" },
+      { "Name": "TVCHOSUN3", "No": "250", "ServiceId": "250" },
+      { "Name": "국악방송", "No": "251", "ServiceId": "251" },
+      { "Name": "한국직업방송", "No": "252", "ServiceId": "252" },
+      { "Name": "토마토클래식", "No": "253", "ServiceId": "253" },
+      { "Name": "WeeTV", "No": "254", "ServiceId": "254" },
+      { "Name": "BALL TV", "No": "255", "ServiceId": "255" },
+      { "Name": "슬로우TV", "No": "256", "ServiceId": "256" },
+      { "Name": "ONT", "No": "257", "ServiceId": "257" },
+      { "Name": "채널칭", "No": "258", "ServiceId": "258" },
+      { "Name": "채널s 플러스", "No": "259", "ServiceId": "259" },
+      { "Name": "국방TV", "No": "260", "ServiceId": "260" },
+      { "Name": "더라이프", "No": "261", "ServiceId": "261" },
+      { "Name": "ONN 닥터TV", "No": "262", "ServiceId": "262" },
+      { "Name": "EBC", "No": "263", "ServiceId": "263" },
+      { "Name": "디스토리", "No": "264", "ServiceId": "264" },
+      { "Name": "RNA", "No": "267", "ServiceId": "267" },
+      { "Name": "리빙TV", "No": "276", "ServiceId": "276" },
+      { "Name": "사회안전방송", "No": "278", "ServiceId": "278" },
+      { "Name": "NBNTV", "No": "285", "ServiceId": "285" }
+    	]
+  },
+  "LG": {
+    "MY_CHANNELS": []
+  },
+  "SK": {
+    "MY_CHANNELS": []
+  },
+  "DAUM": {
+    "MY_CHANNELS": []
+  },
+  "NAVER": {
+    "MY_CHANNELS": []
+  },
+  "WAVVE": {
+    "MY_CHANNELS": [
+    ]
+  },
+  "TVING": {
+    "MY_CHANNELS": []
+  },
+  "SPOTV": {
+    "MY_CHANNELS": []
+  }
+}

epg2xml/epg2xml.sh

@@ -0,0 +1,2 @@
+epg2xml run
+

epg2xml/providers/__init__.py

@@ -0,0 +1,525 @@
+import json
+import logging
+import re
+import sqlite3
+import sys
+from collections import Counter
+from concurrent.futures import ThreadPoolExecutor
+from contextlib import closing
+from dataclasses import astuple, dataclass, fields
+from datetime import datetime, timedelta
+from functools import wraps
+from importlib import import_module
+from itertools import chain
+from os import PathLike
+from typing import Iterator, List, Literal, Tuple, Union
+
+import requests
+
+from epg2xml import __title__, __version__
+from epg2xml.utils import Element, PrefixLogger, RateLimiter, dump_json
+
+log = logging.getLogger("PROV")
+
+
+PTN_TITLE = re.compile(r"(.*) \(?(\d+부)\)?")
+PTN_SPACES = re.compile(r" {2,}")
+CAT_KO2EN = {
+    "교양": "Arts / Culture (without music)",
+    "만화": "Cartoons / Puppets",
+    "교육": "Education / Science / Factual topics",
+    "취미": "Leisure hobbies",
+    "드라마": "Movie / Drama",
+    "영화": "Movie / Drama",
+    "음악": "Music / Ballet / Dance",
+    "뉴스": "News / Current affairs",
+    "다큐": "Documentary",
+    "라이프": "Documentary",
+    "시사/다큐": "Documentary",
+    "연예": "Show / Game show",
+    "스포츠": "Sports",
+    "홈쇼핑": "Advertisement / Shopping",
+}
+TAG_CREDITS = (
+    "director",
+    "actor",
+    "writer",
+    "adapter",
+    "producer",
+    "composer",
+    "editor",
+    "presenter",
+    "commentator",
+    "guest",
+)
+
+
+@dataclass
+class EPGProgram:
+    """For individual program entities"""
+
+    channelid: str
+    stime: datetime = None
+    etime: datetime = None
+    title: str = None
+    title_sub: str = None
+    part_num: str = None
+    ep_num: str = None
+    categories: List[str] = None
+    rebroadcast: bool = False
+    rating: int = 0
+    # not usually given by default
+    desc: str = None
+    poster_url: str = None
+    cast: List[dict] = None  # 출연진
+    crew: List[dict] = None  # 제작진
+    extras: List[str] = None
+    keywords: List[str] = None
+
+    def sanitize(self) -> None:
+        for f in fields(self):
+            attr = getattr(self, f.name)
+            if f.type == List[str] and attr is not None:
+                setattr(self, f.name, [x.strip() for x in filter(bool, attr) if x.strip()])
+            elif f.type == str:
+                setattr(self, f.name, (attr or "").strip())
+
+    def to_xml(self, cfg: dict) -> None:
+        self.sanitize()
+
+        # local variables
+        stime = self.stime.strftime("%Y%m%d%H%M%S +0900")
+        etime = self.etime.strftime("%Y%m%d%H%M%S +0900")
+        title = self.title
+        title_sub = self.title_sub
+        cast = self.cast or []
+        crew = self.crew or []
+        categories = self.categories or []
+        keywords = self.keywords or []
+        episode = self.ep_num
+        rebroadcast = "재" if self.rebroadcast else ""
+        rating = "전체 관람가" if self.rating == 0 else f"{self.rating}세 이상 관람가"
+
+        # programm
+        _p = Element("programme", start=stime, stop=etime, channel=self.channelid)
+
+        # title, sub-title
+        if matches := PTN_TITLE.match(title):
+            title = matches.group(1).strip()
+            title_sub = (matches.group(2) + " " + title_sub).strip()
+        title = [
+            title or title_sub or "제목 없음",
+            f"({episode}회)" if episode and cfg["ADD_EPNUM_TO_TITLE"] else "",
+            f"({rebroadcast})" if rebroadcast and cfg["ADD_REBROADCAST_TO_TITLE"] else "",
+        ]
+        title = PTN_SPACES.sub(" ", " ".join(filter(bool, title)))
+        _p.append(Element("title", title, lang="ko"))
+        if title_sub:
+            _p.append(Element("sub-title", title_sub, lang="ko"))
+
+        # desc
+        if cfg["ADD_DESCRIPTION"]:
+            desc = [
+                title,
+                f"부제 : {title_sub}" if title_sub else "",
+                f"방송 : {rebroadcast}방송" if rebroadcast else "",
+                f"회차 : {episode}회" if episode else "",
+                f"장르 : {','.join(categories)}" if categories else "",
+                f"출연 : {','.join(x['name'] for x in cast)}" if cast else "",
+                f"제작 : {','.join(x['name'] for x in crew)}" if crew else "",
+                f"등급 : {rating}",
+                self.desc,
+            ]
+            desc = PTN_SPACES.sub(" ", "\n".join(filter(bool, desc)))
+            _p.append(Element("desc", desc, lang="ko"))
+
+        # credits
+        if cast or crew:
+            _c = Element("credits")
+            for cc in sorted(cast + crew, key=lambda x: TAG_CREDITS.index(x["title"])):
+                title = cc.pop("title")
+                name = cc.pop("name")
+                _c.append(Element(title, name, **cc))
+            _p.append(_c)
+
+        # categories
+        for cat_ko in categories:
+            _p.append(Element("category", cat_ko, lang="ko"))
+            if cat_en := CAT_KO2EN.get(cat_ko):
+                _p.append(Element("category", cat_en, lang="en"))
+
+        # keywords
+        for keyword in keywords:
+            _p.append(Element("keyword", keyword, lang="ko"))
+
+        # icon
+        if self.poster_url:
+            _p.append(Element("icon", src=self.poster_url))
+
+        # episode-num
+        if episode:
+            if cfg["ADD_XMLTV_NS"]:
+                try:
+                    episode_ns = int(episode) - 1
+                except ValueError:
+                    episode_ns = int(episode.split(",", 1)[0]) - 1
+                episode_ns = f"0.{str(episode_ns)}.0/0"
+                _p.append(Element("episode-num", episode_ns, system="xmltv_ns"))
+            else:
+                _p.append(Element("episode-num", episode, system="onscreen"))
+
+        # previously-shown
+        if rebroadcast:
+            _p.append(Element("previously-shown"))
+
+        # rating
+        if rating:
+            # TODO: 영상물등급위원회(KMRB)는 TV프로그램 심의에 관여하지 않으므로 수정이 필요
+            _r = Element("rating", system="KMRB")
+            _r.append(Element("value", rating))
+            _p.append(_r)
+
+        # dumps
+        print(_p.tostring(level=1))
+
+
+class EPGChannel:
+    """For individual channel entities"""
+
+    __slots__ = ["id", "src", "svcid", "name", "icon", "no", "category", "programs"]
+
+    def __init__(self, channelinfo):
+        self.id: str = channelinfo["Id"]
+        self.src: str = channelinfo["Source"]
+        self.svcid: str = channelinfo["ServiceId"]
+        self.name: str = channelinfo["Name"]
+        self.icon: str = channelinfo.get("Icon_url")
+        self.no: str = channelinfo.get("No")
+        self.category: str = channelinfo.get("Category")
+        # placeholder
+        self.programs: List[EPGProgram] = []
+        """
+        개별 EPGProgram이 소속 channelid를 가지고 있어서 굳이 EPGChannel의 하위 리스트로 관리해야할
+        이유는 없지만, endtime이 없는 EPG 항목을 위해 한 번에 써야할 필요가 있는 Provider가 있기에
+        (kt, lg, skb, naver, daum) 채널 단위로 관리하는 편이 유리하다.
+        """
+
+    def __str__(self):
+        return f"{self.name} <{self.id}>"
+
+    def set_etime(self) -> None:
+        """Completes missing program endtimes based on the successive relationship between programs."""
+        for ind, prog in enumerate(self.programs):
+            if prog.etime:
+                continue
+            try:
+                prog.etime = self.programs[ind + 1].stime
+            except IndexError:
+                prog.etime = (prog.stime + timedelta(days=1)).replace(hour=0, minute=0, second=0)
+
+    def to_xml(self) -> None:
+        chel = Element("channel", id=self.id)
+        # TODO: something better for display-name?
+        chel.append(Element("display-name", self.name))
+        chel.append(Element("display-name", self.src))
+        if self.no:
+            chel.append(Element("display-name", f"{self.no}"))
+            chel.append(Element("display-name", f"{self.no} {self.name}"))
+            chel.append(Element("display-name", f"{self.no} {self.src}"))
+        if self.icon:
+            chel.append(Element("icon", src=self.icon))
+        print(chel.tostring(level=1))
+
+
+UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36"
+
+
+class EPGProvider:
+    """Base class for EPG Providers"""
+
+    referer: str = None
+    title_regex: Union[str, re.Pattern] = None
+    tps: float = 1.0
+    was_channel_updated: bool = False
+
+    def __init__(self, cfg: dict):
+        self.provider_name = self.__class__.__name__
+        self.cfg = cfg
+        self.sess = requests.Session()
+        self.sess.headers.update({"User-Agent": UA, "Referer": self.referer})
+        if http_proxy := cfg["HTTP_PROXY"]:
+            self.sess.proxies.update({"http": http_proxy, "https": http_proxy})
+        if self.title_regex:
+            self.title_regex = re.compile(self.title_regex)
+        self.request = RateLimiter(tps=self.tps)(self.__request)
+        # placeholders
+        self.svc_channels: List[dict] = []
+        self.req_channels: List[EPGChannel] = []
+
+    def __request(self, url: str, method: str = "GET", **kwargs) -> str:
+        ret = ""
+        try:
+            r = self.sess.request(method=method, url=url, **kwargs)
+            try:
+                ret = r.json()
+            except (json.decoder.JSONDecodeError, ValueError):
+                ret = r.text
+        except requests.exceptions.HTTPError as e:
+            log.error("요청 중 에러: %s", e)
+        except Exception:
+            log.exception("요청 중 예외:")
+        return ret
+
+    def load_svc_channels(self, channeljson: dict = None) -> None:
+        plog = PrefixLogger(log, f"[{self.provider_name:5s}]")
+
+        # check if update required
+        try:
+            channelinfo = channeljson[self.provider_name.upper()]
+            total = channelinfo["TOTAL"]
+            channels = channelinfo["CHANNELS"]
+            assert total == len(channels), "TOTAL != len(CHANNELS)"
+            updated_at = datetime.fromisoformat(channelinfo["UPDATED"])
+            if (datetime.now() - updated_at).total_seconds() <= 3600 * 24 * 4:
+                self.svc_channels = channels
+                plog.info("%03d service channels loaded from cache", len(channels))
+                return
+            plog.debug("Updating service channels as outdated...")
+        except Exception as e:
+            plog.debug("Updating service channels as cache broken: %s", e)
+
+        try:
+            channels = self.get_svc_channels()
+        except Exception:
+            plog.exception("Exception while retrieving service channels:")
+        else:
+            self.svc_channels = channels
+            self.was_channel_updated = True
+            plog.info("%03d service channels successfully fetched from server", len(channels))
+
+    def get_svc_channels(self) -> List[dict]:
+        raise NotImplementedError("method 'get_svc_channels' must be implemented")
+
+    def load_req_channels(self) -> None:
+        """from MY_CHANNELS to req_channels"""
+        plog = PrefixLogger(log, f"[{self.provider_name:5s}]")
+        my_channels = self.cfg["MY_CHANNELS"]
+        if my_channels == "*":
+            plog.debug("Overriding all MY_CHANNELS by service channels...")
+            my_channels = self.svc_channels
+        if not my_channels:
+            return
+        req_channels = []
+        svc_channels = {x["ServiceId"]: x for x in self.svc_channels}
+        for my_no, my_ch in enumerate(my_channels):
+            if "ServiceId" not in my_ch:
+                plog.warning("'ServiceId' Not Found: %s", my_ch)
+                continue
+            req_ch = svc_channels.pop(my_ch["ServiceId"], None)
+            if req_ch is None:
+                plog.warning("'ServiceId' Not in Service: %s", my_ch)
+                continue
+            for _k, _v in my_ch.items():
+                if _v:
+                    req_ch[_k] = _v
+            req_ch["Source"] = self.provider_name
+            req_ch.setdefault("No", str(my_no))
+            if "Id" not in req_ch:
+                try:
+                    req_ch["Id"] = eval(f"f'{self.cfg['ID_FORMAT']}'", None, req_ch)
+                except Exception:
+                    req_ch["Id"] = f'{req_ch["ServiceId"]}.{req_ch["Source"].lower()}'
+            if not self.cfg["ADD_CHANNEL_ICON"]:
+                req_ch.pop("Icon_url", None)
+            req_channels.append(EPGChannel(req_ch))
+        plog.info("요청 %3d - 불가 %3d = 최종 %3d", len(my_channels), len(my_channels) - len(req_channels), len(req_channels))
+        self.req_channels = req_channels
+
+    def write_channels(self) -> None:
+        for ch in self.req_channels:
+            if not ch.programs:
+                log.warning("Skip writing as no program entries found for '%s'", ch.id)
+                continue
+            ch.to_xml()
+
+    def get_programs(self) -> None:
+        raise NotImplementedError("method 'get_programs' must be implemented")
+
+    def write_programs(self) -> None:
+        for ch in self.req_channels:
+            for prog in ch.programs:
+                prog.to_xml(self.cfg)
+            ch.programs.clear()  # for memory efficiency
+
+
+def no_endtime(func):
+    @wraps(func)
+    def wrapped(self: EPGProvider, *args, **kwargs):
+        func(self, *args, **kwargs)
+        for ch in self.req_channels:
+            ch.set_etime()
+
+    return wrapped
+
+
+class EPGHandler:
+    """for handling EPGProviders"""
+
+    def __init__(self, cfgs: dict):
+        self.providers: List[EPGProvider] = self.load_providers(cfgs)
+
+    def load_providers(self, cfgs: dict) -> List[EPGProvider]:
+        providers = []
+        for name, cfg in cfgs.items():
+            if not cfg["ENABLED"]:
+                continue
+            try:
+                m = import_module(f"epg2xml.providers.{name.lower()}")
+                providers.append(getattr(m, name.upper())(cfg))
+            except ModuleNotFoundError:
+                log.error("No such provider found: '%s'", name)
+                sys.exit(1)
+        return providers
+
+    def load_channels(self, channelfile: str, parallel: bool = False) -> None:
+        try:
+            log.debug("Trying to load cached channels from json")
+            with open(channelfile, "r", encoding="utf-8") as fp:
+                channeljson = json.load(fp)
+        except (json.decoder.JSONDecodeError, ValueError, FileNotFoundError) as e:
+            log.debug("Failed to load cached channels from json: %s", e)
+            channeljson = {}
+        if parallel:
+            with ThreadPoolExecutor() as exe:
+                for p in self.providers:
+                    exe.submit(p.load_svc_channels, channeljson=channeljson)
+        else:
+            for p in self.providers:
+                p.load_svc_channels(channeljson=channeljson)
+        if any(p.was_channel_updated for p in self.providers):
+            for p in self.providers:
+                channeljson[p.provider_name.upper()] = {
+                    "UPDATED": datetime.now().isoformat(),
+                    "TOTAL": len(p.svc_channels),
+                    "CHANNELS": p.svc_channels,
+                }
+            dump_json(channelfile, channeljson)
+            log.info("Channel file was upgraded. You may check the changes here: %s", channelfile)
+
+    def load_req_channels(self):
+        for p in self.providers:
+            p.load_req_channels()
+
+        log.debug("Checking uniqueness of channelid...")
+        cids = [c.id for p in self.providers for c in p.req_channels]
+        assert len(cids) == len(set(cids)), f"채널ID 중복: { {k:v for k,v in Counter(cids).items() if v > 1} }"
+
+    def get_programs(self, parallel: bool = False):
+        if parallel:
+            with ThreadPoolExecutor() as exe:
+                for p in self.providers:
+                    exe.submit(p.get_programs)
+        else:
+            for p in self.providers:
+                p.get_programs()
+
+    def to_xml(self):
+        print('<?xml version="1.0" encoding="UTF-8"?>')
+        print('<!DOCTYPE tv SYSTEM "xmltv.dtd">\n')
+        print(f'<tv generator-info-name="{__title__} v{__version__}">')
+
+        log.debug("Writing channels...")
+        for p in self.providers:
+            p.write_channels()
+
+        log.debug("Writing programs...")
+        for p in self.providers:
+            p.write_programs()
+
+        print("</tv>")
+
+    @property
+    def all_channels(self) -> Iterator:
+        """shortcut to access all channels in providers"""
+        return chain.from_iterable(p.req_channels for p in self.providers)
+
+    @property
+    def all_programs(self) -> Iterator:
+        """shortcut to access all programs in providers"""
+        return chain.from_iterable(ch.programs for ch in self.all_channels)
+
+    def to_db(self, dbfile: PathLike) -> None:
+        with SQLite(dbfile, "w") as db:
+            db.insert_channels(self.all_channels)
+            db.insert_programs(self.all_programs)
+
+    def from_db(self, dbfile: PathLike) -> None:
+        with SQLite(dbfile, "r") as db:
+            for p in self.providers:
+                for ch in db.queryall("SELECT * FROM epgchannel WHERE Source = ?", (p.provider_name,)):
+                    chn = EPGChannel(dict(ch))
+                    for prog in db.queryall("SELECT * FROM epgprogram WHERE channelid = ?", (chn.id,)):
+                        chn.programs.append(EPGProgram(**dict(prog)))
+                    p.req_channels.append(chn)
+
+
+sqlite3.register_adapter(bool, int)
+sqlite3.register_converter("BOOLEAN", lambda v: bool(int(v)))
+sqlite3.register_adapter(list, lambda v: json.dumps(v, ensure_ascii=False))
+sqlite3.register_converter("JSON", json.loads)
+
+SQLITE_DTYPES = {
+    bool: "BOOLEAN",
+    datetime: "TIMESTAMP",
+    int: "INTEGER",
+    List[dict]: "JSON",
+    List[str]: "JSON",
+}
+
+
+class SQLite:
+    def __init__(self, dbfile: PathLike, mode: Literal["r", "w", "a"] = "r", **kwargs):
+        kwargs.setdefault("detect_types", sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES)
+        self.conn = sqlite3.connect(dbfile, **kwargs)
+        self.conn.row_factory = sqlite3.Row
+        self.__db_init(mode=mode)
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.conn.close()
+
+    def __db_init(self, mode: Literal["r", "w", "a"]) -> None:
+        if mode == "r":
+            return
+        with closing(self.conn.cursor()) as c:
+            # create table - epgchannel
+            c.execute("CREATE TABLE IF NOT EXISTS epgchannel (Id, Source, ServiceId, Name, Icon_url, No, Category)")
+            # create table - epgprogram
+            cols = [f"{f.name} {SQLITE_DTYPES.get(f.type, 'TEXT')}" for f in fields(EPGProgram)]
+            c.execute(f"CREATE TABLE IF NOT EXISTS epgprogram ({', '.join(cols)})")
+            if mode == "w":
+                c.execute("DELETE FROM epgchannel")
+                c.execute("DELETE FROM epgprogram")
+        self.conn.commit()
+
+    def insert_channels(self, channels: List[EPGChannel]) -> None:
+        def _astuple(ch: EPGChannel) -> Tuple:
+            return (ch.id, ch.src, ch.svcid, ch.name, ch.icon, ch.no, ch.category)
+
+        sql = "INSERT INTO epgchannel VALUES (?,?,?,?,?,?,?)"
+        with closing(self.conn.cursor()) as c:
+            c.executemany(sql, map(_astuple, channels))
+        self.conn.commit()
+
+    def insert_programs(self, programs: List[EPGProgram]) -> None:
+        cols = [f.name for f in fields(EPGProgram)]
+        sql = f"INSERT INTO epgprogram({','.join(cols)}) VALUES ({','.join('?'*len(cols))})"
+        with closing(self.conn.cursor()) as c:
+            c.executemany(sql, map(astuple, programs))
+        self.conn.commit()
+
+    def queryall(self, *args, **kwargs) -> List[sqlite3.Row]:
+        with closing(self.conn.cursor()) as c:
+            return c.execute(*args, **kwargs).fetchall()

epg2xml/providers/daum.py

@@ -0,0 +1,109 @@
+import logging
+from datetime import datetime, timedelta
+from typing import List
+from urllib.parse import quote
+
+from epg2xml.providers import EPGProgram, EPGProvider, no_endtime
+from epg2xml.utils import ParserBeautifulSoup as BeautifulSoup
+
+log = logging.getLogger(__name__.rsplit(".", maxsplit=1)[-1].upper())
+
+CH_CATE = ["지상파", "종합편성", "케이블", "스카이라이프", "해외위성", "라디오"]
+
+
+class DAUM(EPGProvider):
+    """EPGProvider for DAUM
+
+    데이터: rawhtml
+    요청수: #channels
+    특이사항:
+    - 최대 7일치를 한 번에
+    - 프로그램 시작 시각만 제공
+    """
+
+    referer = None
+    title_regex = r"^(?P<title>.*?)\s?([\<\(]?(?P<part>\d{1})부[\>\)]?)?\s?(<(?P<subname1>.*)>)?\s?((?P<epnum>\d+)회)?\s?(<(?P<subname2>.*)>)?$"
+
+    def get_svc_channels(self) -> List[dict]:
+        svc_channels = []
+        url = "https://search.daum.net/search?DA=B3T&w=tot&rtmaxcoll=B3T&q={}"
+        channelsel1 = '#channelNaviLayer > div[class^="layer_tv layer_all"] ul > li'
+        channelsel2 = 'div[class="wrap_sub"] > span > a'
+        for c in CH_CATE:
+            search_url = url.format(f"{c} 편성표")
+            data = self.request(search_url)
+            soup = BeautifulSoup(data)
+            if not soup.find_all(attrs={"disp-attr": "B3T"}):
+                continue
+            all_channels = [str(x.text.strip()) for x in soup.select(channelsel1)]
+            if not all_channels:
+                all_channels += [str(x.text.strip()) for x in soup.select(channelsel2)]
+            svc_cate = c.replace("스카이라이프", "SKYLIFE")
+            for x in all_channels:
+                svc_channels.append(
+                    {
+                        "Name": x,
+                        "ServiceId": f"{svc_cate} {x}",
+                        "Category": c,
+                    }
+                )
+        return svc_channels
+
+    @no_endtime
+    def get_programs(self) -> None:
+        url = "https://search.daum.net/search?DA=B3T&w=tot&rtmaxcoll=B3T&q={}"
+        for idx, _ch in enumerate(self.req_channels):
+            log.info("%03d/%03d %s", idx + 1, len(self.req_channels), _ch)
+            search_url = url.format(quote(_ch.svcid + " 편성표"))
+            data = self.request(search_url)
+            try:
+                _epgs = self.__epgs_of_days(_ch.id, data)
+            except AssertionError as e:
+                log.warning("%s: %s", e, _ch)
+            except Exception:
+                log.exception("프로그램 파싱 중 예외: %s", _ch)
+            else:
+                _ch.programs.extend(_epgs)
+
+    def __epgs_of_days(self, channelid: str, data: str) -> List[EPGProgram]:
+        soup = BeautifulSoup(data)
+        assert soup.find_all(attrs={"disp-attr": "B3T"}), "EPG 정보가 없거나 없는 채널입니다"
+        days = soup.select('div[class="tbl_head head_type2"] > span > span[class="date"]')
+
+        # 연도 추정
+        currdate = datetime.now()  # 언제나 basedate보다 미래
+        basedate = datetime.strptime(days[0].text.strip(), "%m.%d").replace(year=currdate.year)
+        if (basedate - currdate).days > 0:
+            basedate = basedate.replace(year=basedate.year - 1)
+
+        _epgs = []
+        for nd, _ in enumerate(days):
+            hours = soup.select(f'[id="tvProgramListWrap"] > table > tbody > tr > td:nth-of-type({nd+1})')
+            assert len(hours) == 24, f"24개의 시간 행이 있어야 합니다: 현재: {len(hours):d}"
+            for nh, hour in enumerate(hours):
+                for dl in hour.select("dl"):
+                    _epg = EPGProgram(channelid)
+                    nm = int(dl.select("dt")[0].text.strip())
+                    _epg.stime = basedate + timedelta(days=nd, hours=nh, minutes=nm)
+                    for atag in dl.select("dd > a"):
+                        _epg.title = atag.text.strip()
+                    for span in dl.select("dd > span"):
+                        class_val = " ".join(span["class"])
+                        if class_val == "":
+                            _epg.title = span.text.strip()
+                        elif "ico_re" in class_val:
+                            _epg.rebroadcast = True
+                        elif "ico_rate" in class_val:
+                            _epg.rating = int(class_val.split("ico_rate")[1].strip())
+                        else:
+                            # ico_live ico_hd ico_subtitle ico_hand ico_uhd ico_talk ico_st
+                            _epg.extras = (_epg.extras or []) + [span.text.strip()]
+                    if m := self.title_regex.search(_epg.title):
+                        _epg.title = m.group("title")
+                        _epg.part_num = m.group("part")
+                        _epg.ep_num = m.group("epnum")
+                        _epg.title_sub = m.group("subname2") or m.group("subname1")
+                        if _epg.part_num:
+                            _epg.title += f" {_epg.part_num}부"
+                    _epgs.append(_epg)
+        return _epgs

epg2xml/providers/kt.py

@@ -0,0 +1,116 @@
+import logging
+import re
+from datetime import date, datetime, timedelta
+from typing import List
+from urllib.parse import unquote
+
+from bs4 import SoupStrainer
+
+from epg2xml.providers import EPGProgram, EPGProvider, no_endtime
+from epg2xml.utils import ParserBeautifulSoup as BeautifulSoup
+
+log = logging.getLogger(__name__.rsplit(".", maxsplit=1)[-1].upper())
+
+CH_CATE = [
+    # 0은 전체 채널
+    {"id": "1", "name": "UHD"},
+    {"id": "3", "name": "홍보"},
+    {"id": "4", "name": "지상파"},
+    {"id": "5", "name": "홈쇼핑"},
+    {"id": "6", "name": "종합편성"},
+    {"id": "8", "name": "드라마/버라이어티"},
+    {"id": "10", "name": "오락/음악"},
+    {"id": "12", "name": "영화/시리즈"},
+    {"id": "137", "name": "스포츠"},
+    {"id": "206", "name": "취미/레저"},
+    {"id": "317", "name": "애니/유아"},
+    {"id": "442", "name": "교육"},
+    {"id": "446", "name": "다큐/교양"},
+    {"id": "447", "name": "뉴스/경제"},
+    {"id": "448", "name": "공공/공익/정보"},
+    {"id": "449", "name": "종교"},
+    {"id": "491", "name": "오픈채널"},
+    {"id": "507", "name": "유료"},
+    {"id": "508", "name": "오디오"},
+]
+PTN_RATING = re.compile(r"([\d,]+)")
+
+
+class KT(EPGProvider):
+    """EPGProvider for KT
+
+    데이터: rawhtml
+    요청수: #channels * #days
+    특이사항:
+    - 가끔 업데이트 지연
+    - 프로그램 시작 시각만 제공
+    """
+
+    referer = "https://tv.kt.com/"
+    title_regex = r"^(?P<title>.*?)\s?([\<\(]?(?P<part>\d+)부[\>\)]?)?$"
+
+    def get_svc_channels(self) -> List[dict]:
+        svc_channels = []
+        url = "https://tv.kt.com/tv/channel/pChList.asp"
+        params = {"ch_type": "1", "parent_menu_id": "0"}
+        for c in CH_CATE:
+            params.update({"parent_menu_id": c["id"]})
+            soup = BeautifulSoup(self.request(url, method="POST", data=params))
+            raw_channels = [unquote(x.find("span", {"class": "ch"}).text.strip()) for x in soup.select("li > a")]
+            # 몇몇 채널은 (TV로만 제공, 유료채널) 웹에서 막혀있지만 실제로는 데이터가 있을 수 있다.
+            for x in raw_channels:
+                svc_channels.append(
+                    {
+                        "Name": " ".join(x.split()[1:]),
+                        "No": str(x.split()[0]),
+                        "ServiceId": x.split()[0],
+                        "Category": c["name"],
+                    }
+                )
+        return svc_channels
+
+    @no_endtime
+    def get_programs(self) -> None:
+        url = "https://tv.kt.com/tv/channel/pSchedule.asp"
+        params = {
+            "ch_type": "1",  # 1: live 2: skylife 3: uhd live 4: uhd skylife
+            "view_type": "1",  # 1: daily 2: weekly
+            "service_ch_no": "SVCID",
+            "seldate": "EPGDATE",
+        }
+        for idx, _ch in enumerate(self.req_channels):
+            log.info("%03d/%03d %s", idx + 1, len(self.req_channels), _ch)
+            for nd in range(int(self.cfg["FETCH_LIMIT"])):
+                day = date.today() + timedelta(days=nd)
+                params.update({"service_ch_no": _ch.svcid, "seldate": day.strftime("%Y%m%d")})
+                data = self.request(url, method="POST", data=params)
+                try:
+                    _epgs = self.__epgs_of_day(_ch.id, data, day)
+                except Exception:
+                    log.exception("프로그램 파싱 중 예외: %s, %s", _ch, day)
+                else:
+                    _ch.programs.extend(_epgs)
+
+    def __epgs_of_day(self, channelid: str, data: str, day: datetime) -> List[EPGProgram]:
+        _epgs = []
+        soup = BeautifulSoup(unquote(data), parse_only=SoupStrainer("tbody"))
+        for row in soup.find_all("tr"):
+            cell = row.find_all("td")
+            hour = cell[0].text.strip()
+            for minute, program, category in zip(*[c.find_all("p") for c in cell[1:]]):
+                _epg = EPGProgram(channelid)
+                _epg.stime = datetime.strptime(f"{day} {hour}:{minute.text.strip()}", "%Y-%m-%d %H:%M")
+                _epg.title = program.text.replace("방송중 ", "").strip()
+                if m := self.title_regex.match(_epg.title):
+                    _epg.title = m.group("title")
+                    if part_num := m.group("part"):
+                        _epg.part_num = part_num
+                        _epg.title += f" ({_epg.part_num}부)"
+                _epg.categories = [category.text.strip()]
+                for image in program.find_all("img", alt=True):
+                    if "시청 가능" not in (alt := image["alt"]):
+                        continue
+                    grade = PTN_RATING.match(alt)
+                    _epg.rating = int(grade.group(1)) if grade else 0
+                _epgs.append(_epg)
+        return _epgs

epg2xml/providers/lg.py

@@ -0,0 +1,118 @@
+import logging
+from datetime import date, datetime, timedelta
+from typing import List
+
+from epg2xml.providers import EPGProgram, EPGProvider, no_endtime
+
+log = logging.getLogger(__name__.rsplit(".", maxsplit=1)[-1].upper())
+
+G_CODE = {"0": 0, "1": 7, "2": 12, "3": 15, "4": 19}
+P_CATE = {
+    "00": "영화",
+    "02": "만화",
+    "03": "드라마",
+    "05": "스포츠",
+    "06": "교육",
+    "07": None,  # 어린이/교육
+    "08": "연예/오락",
+    "09": "공연/음악",
+    "10": None,  # 게임
+    "11": "다큐",
+    "12": "뉴스/정보",
+    "13": "라이프",
+    "15": None,  # 홈쇼핑
+    "16": None,  # 경제/부동산
+    "31": "기타",
+}
+
+
+class LG(EPGProvider):
+    """EPGProvider for LG
+
+    데이터: jsonapi
+    요청수: #channels * #days
+    특이사항:
+    - 5일치만 제공
+    - 프로그램 시작 시각만 제공
+    참고:
+    - 사이트 리뉴얼 이후 프로그램 카테고리가 아직 명확히 정해지지 않은 듯 하다.
+    """
+
+    referer = "https://www.lguplus.com/iptv/channel-guide"
+    title_regex = r"\s?(?:\[.*?\])?(.*?)(?:\[(.*)\])?\s?(?:\(([\d,]+)회\))?\s?(<재>)?$"
+
+    def get_svc_channels(self) -> List[dict]:
+        svc_channels = []
+        url = "https://www.lguplus.com/uhdc/fo/prdv/chnlgid/v1/tv-schedule-list"
+        data = self.request(url)
+        cate = {x["urcBrdCntrTvChnlGnreCd"]: x["urcBrdCntrTvChnlGnreNm"] for x in data["brdGnreDtoList"]}
+        for ch in self.request(url)["brdCntrTvChnlIDtoList"]:
+            svc_channels.append(
+                {
+                    "Name": ch["urcBrdCntrTvChnlDscr"],
+                    "No": ch["urcBrdCntrTvChnlNo"],
+                    "ServiceId": ch["urcBrdCntrTvChnlId"],
+                    "Category": cate[ch["urcBrdCntrTvChnlGnreCd"]],
+                }
+            )
+        return svc_channels
+
+    @no_endtime
+    def get_programs(self) -> None:
+        max_ndays = 5
+        if int(self.cfg["FETCH_LIMIT"]) > max_ndays:
+            log.warning(
+                """
+
+***********************************************************************
+
+%s는 당일포함 %d일치만 EPG를 제공하고 있습니다.
+
+***********************************************************************
+            """,
+                self.provider_name,
+                max_ndays,
+            )
+        url = "https://www.lguplus.com/uhdc/fo/prdv/chnlgid/v1/tv-schedule-list"
+        params = {"urcBrdCntrTvChnlId": "SVCID", "brdCntrTvChnlBrdDt": "EPGDATE"}
+        for idx, _ch in enumerate(self.req_channels):
+            log.info("%03d/%03d %s", idx + 1, len(self.req_channels), _ch)
+            for nd in range(min(int(self.cfg["FETCH_LIMIT"]), max_ndays)):
+                day = date.today() + timedelta(days=nd)
+                params.update({"urcBrdCntrTvChnlId": _ch.svcid, "brdCntrTvChnlBrdDt": day.strftime("%Y%m%d")})
+                data = self.request(url, params=params) or {}
+                data = data.get("brdCntTvSchIDtoList", [])
+                if not data:
+                    log.warning("EPG 정보가 없거나 없는 채널입니다: %s %s", _ch, day)
+                    break  # 오늘 없으면 내일도 없는 채널로 간주
+                try:
+                    _epgs = self.__epgs_of_day(_ch.id, data)
+                except Exception:
+                    log.exception("프로그램 파싱 중 예외: %s, %s", _ch, day)
+                else:
+                    _ch.programs.extend(_epgs)
+
+    def __epgs_of_day(self, channelid: str, data: list) -> List[EPGProgram]:
+        _epgs = []
+        for p in data:
+            _epg = EPGProgram(channelid)
+            _epg.title = p["brdPgmTitNm"]
+            _epg.desc = p["brdPgmDscr"]
+            _epg.stime = datetime.strptime(p["brdCntrTvChnlBrdDt"] + p["epgStrtTme"], "%Y%m%d%H:%M:%S")
+            _epg.rating = G_CODE.get(p["brdWtchAgeGrdCd"], 0)
+            _epg.extras = [p["brdPgmRsolNm"]]  # 화질
+            if p["subtBrdYn"] == "Y":
+                _epg.extras.append("자막")
+            if p["explBrdYn"] == "Y":
+                _epg.extras.append("화면해설")
+            if p["silaBrdYn"] == "Y":
+                _epg.extras.append("수화")
+            if m := self.title_regex.match(_epg.title):
+                _epg.title = m.group(1)
+                _epg.title_sub = m.group(2)
+                _epg.ep_num = m.group(3)
+                _epg.rebroadcast = bool(m.group(4))
+            if P_CATE[p["urcBrdCntrTvSchdGnreCd"]]:
+                _epg.categories = [P_CATE[p["urcBrdCntrTvSchdGnreCd"]]]
+            _epgs.append(_epg)
+        return _epgs

epg2xml/providers/naver.py

@@ -0,0 +1,106 @@
+import logging
+from datetime import date, datetime, timedelta
+from typing import List
+from xml.sax.saxutils import unescape
+
+from epg2xml.providers import EPGProgram, EPGProvider, no_endtime
+from epg2xml.utils import ParserBeautifulSoup as BeautifulSoup
+
+log = logging.getLogger(__name__.rsplit(".", maxsplit=1)[-1].upper())
+today = date.today()
+
+CH_CATE = [
+    {"name": "지상파", "u1": "100"},
+    {"name": "종합 편성", "u1": "500"},
+    {"name": "케이블", "u1": "200"},
+    {"name": "스카이라이프", "u1": "300"},
+    {"name": "해외위성", "u1": "9000"},
+    {"name": "라디오", "u1": "400"},
+]
+
+
+class NAVER(EPGProvider):
+    """EPGProvider for NAVER
+
+    데이터: rawhtml
+    요청수: #channels * #days
+    특이사항:
+    - 프로그램 시작 시각만 제공
+    """
+
+    referer = "https://m.search.naver.com/search.naver?where=m&query=%ED%8E%B8%EC%84%B1%ED%91%9C"
+
+    def get_svc_channels(self) -> List[dict]:
+        svc_channels = []
+        url = "https://m.search.naver.com/p/csearch/content/nqapirender.nhn"
+        params = {
+            "key": "ScheduleChannelList",
+            "where": "nexearch",
+            "pkid": "66",
+            "u1": "CATEGORY_CODE",
+        }
+        for c in CH_CATE:
+            params.update({"u1": c["u1"]})
+            data = self.request(url, params=params)
+            if data["statusCode"].lower() != "success":
+                log.error("유효한 응답이 아닙니다: %s", data["statusCode"])
+                continue
+            soup = BeautifulSoup(data["dataHtml"])
+            for ch in soup.select('li[class="item"]'):
+                try:
+                    svcid = ch.select("div > div[data-cid]")[0]["data-cid"]
+                    name = str(ch.select('div[class="channel_name"] > a')[0].text)
+                    svc_channels.append(
+                        {
+                            "Name": name,
+                            "ServiceId": svcid,
+                            "Category": c["name"],
+                        }
+                    )
+                except Exception:
+                    pass
+        return svc_channels
+
+    @no_endtime
+    def get_programs(self) -> None:
+        url = "https://m.search.naver.com/p/csearch/content/nqapirender.nhn"
+        params = {"key": "SingleChannelDailySchedule", "where": "m", "pkid": "66", "u1": "SVCID", "u2": "EPGDATE"}
+
+        for idx, _ch in enumerate(self.req_channels):
+            log.info("%03d/%03d %s", idx + 1, len(self.req_channels), _ch)
+            for nd in range(int(self.cfg["FETCH_LIMIT"])):
+                day = today + timedelta(days=nd)
+                params.update({"u1": _ch.svcid, "u2": day.strftime("%Y%m%d")})
+                data = self.request(url, params=params)
+                if data["statusCode"].lower() != "success":
+                    log.error("유효한 응답이 아닙니다: %s %s", _ch, data["statusCode"])
+                    continue
+                try:
+                    _epgs = self.__epgs_of_day(_ch.id, data, day)
+                except Exception:
+                    log.exception("프로그램 파싱 중 예외: %s, %s", _ch, day)
+                else:
+                    _ch.programs.extend(_epgs)
+
+    def __epgs_of_day(self, channelid: str, data: dict, day: datetime) -> List[EPGProgram]:
+        _epgs = []
+        soup = BeautifulSoup("".join(data["dataHtml"]))
+        for row in soup.find_all("li", {"class": "list"}):
+            cell = row.find_all("div")
+            _epg = EPGProgram(channelid)
+            _epg.title = unescape(cell[4].text.strip())
+            _epg.stime = datetime.strptime(f"{str(day)} {cell[1].text.strip()}", "%Y-%m-%d %H:%M")
+            for span in cell[3].findAll("span"):
+                span_txt = span.text.strip()
+                if "ico_age" in span["class"]:
+                    _epg.rating = int(span_txt.rstrip("세"))
+                elif "re" in span["class"]:
+                    _epg.rebroadcast = True
+                else:
+                    _epg.extras = (_epg.extras or []) + [span_txt]
+            try:
+                _epg.title_sub = cell[5].text.strip()
+            except Exception:
+                pass
+            _epgs.append(_epg)
+        return _epgs

epg2xml/providers/sk.py

@@ -0,0 +1,112 @@
+import logging
+from datetime import date, datetime, timedelta
+from typing import List
+from xml.sax.saxutils import unescape
+
+from epg2xml.providers import EPGProgram, EPGProvider
+
+log = logging.getLogger(__name__.rsplit(".", maxsplit=1)[-1].upper())
+
+GENRE_CODE = {
+    "1": "드라마",
+    "2": "영화",
+    "4": "만화",
+    "8": "스포츠",
+    "9": "교육",
+    "11": "홈쇼핑",
+    "13": "예능",
+    "14": "시사/다큐",
+    "15": "음악",
+    "16": "라이프",
+    "17": "교양",
+    "18": "뉴스",
+}
+
+
+class SK(EPGProvider):
+    """EPGProvider for SK
+
+    데이터: jsonapi
+    요청수: #channels
+    특이사항:
+    - 최대 3일치를 한 번에
+    """
+
+    referer = "https://www.bworld.co.kr/"
+    title_regex = r"^(.*?)(\(([\d,]+)회\))?(<(.*)>)?(\((재)\))?$"
+
+    def get_svc_channels(self) -> List[dict]:
+        svc_channels = []
+        url = "https://www.bworld.co.kr/content/realtime/realtime_list.ajax"
+        params = {"pack": "PM50305785"}
+        c_name = ""
+        for x in self.request(url, params=params):
+            if x["depth"] == "1":
+                c_name = x["m_name"]
+            elif x["depth"] == "2" and c_name and c_name not in ["프로모션"]:
+                svc_channels.append(
+                    {
+                        "Name": unescape(x["m_name"]),
+                        "No": str(x["ch_no"]),
+                        "ServiceId": x["id_svc"],
+                        "Category": c_name,
+                    }
+                )
+        return svc_channels
+
+    def get_programs(self) -> None:
+        max_ndays = 3
+        if int(self.cfg["FETCH_LIMIT"]) > max_ndays:
+            log.warning(
+                """
+
+***********************************************************************
+
+%s는 당일포함 %d일치만 EPG를 제공하고 있습니다.
+
+***********************************************************************
+        """,
+                self.provider_name,
+                max_ndays,
+            )
+        url = "https://www.bworld.co.kr/myb/core-prod/product/btv-channel/week-frmt-list"
+        params = {"idSvc": "SVCID", "stdDt": "EPGDATE", "gubun": "week"}
+
+        for idx, _ch in enumerate(self.req_channels):
+            log.info("%03d/%03d %s", idx + 1, len(self.req_channels), _ch)
+            params.update({"idSvc": _ch.svcid, "stdDt": date.today().strftime("%Y%m%d")})
+            try:
+                infolist = self.request(url, params=params)["result"]["chnlFrmtInfoList"]
+                assert isinstance(infolist, list)
+            except Exception:
+                log.exception("예상치 못한 응답: %s", params)
+                continue
+            for nd in range(min(int(self.cfg["FETCH_LIMIT"]), max_ndays)):
+                day = date.today() + timedelta(days=nd)
+                try:
+                    _epgs = self.__epgs_of_day(_ch.id, infolist, day)
+                except Exception:
+                    log.exception("프로그램 파싱 중 예외: %s, %s", _ch, day)
+                else:
+                    _ch.programs.extend(_epgs)
+
+    def __epgs_of_day(self, channelid: str, data: list, day: datetime) -> List[EPGProgram]:
+        _epgs = []
+        for info in data:
+            if info["eventDt"] != day.strftime("%Y%m%d"):
+                continue
+            _epg = EPGProgram(channelid)
+            _epg.title = info["nmTitle"]
+            if m := self.title_regex.match(_epg.title):
+                _epg.title = m.group(1)
+                _epg.title_sub = m.group(5)
+                _epg.rebroadcast = bool(m.group(7))
+                _epg.ep_num = m.group(3)
+            _epg.rating = int(info.get("cdRating") or "0")
+            _epg.stime = datetime.strptime(info["dtEventStart"], "%Y%m%d%H%M%S")
+            _epg.etime = datetime.strptime(info["dtEventEnd"], "%Y%m%d%H%M%S")
+            if info["cdGenre"] and (info["cdGenre"] in GENRE_CODE):
+                _epg.categories = [GENRE_CODE[info["cdGenre"]]]
+            _epg.desc = info["nmSynop"] or None  # 값이 없음
+            _epgs.append(_epg)
+        return _epgs

epg2xml/providers/spotv.py

@@ -0,0 +1,114 @@
+import logging
+from datetime import date, datetime, timedelta
+from typing import List
+
+from epg2xml.providers import EPGProgram, EPGProvider
+
+log = logging.getLogger(__name__.rsplit(".", maxsplit=1)[-1].upper())
+
+
+class SPOTV(EPGProvider):
+    """EPGProvider for SPOTV
+
+    데이터: jsonapi
+    요청수: #days
+    특이사항:
+    - 5일치만 제공
+    """
+
+    referer = "https://www.spotvnow.co.kr/channel"
+    title_regex = r"\s?(?:\[(.*?)\])?\s?(.*?)\s?(?:[\(<](.*)[\)>])?\s?(?:-(\d+))?\s?(?:<?([\d,]+)회>?)?\s?$"
+
+    def get_svc_channels(self) -> List[dict]:
+        url = "https://www.spotvnow.co.kr/api/v3/channel"
+        return [
+            {
+                "Name": ch["name"],
+                "ServiceId": ch["id"],
+                "Icon_url": ch["logo"],
+            }
+            for ch in self.request(url)
+        ]
+
+    def __dt(self, dt: str) -> datetime:
+        if not dt:
+            return None
+        if dt.endswith("24:00"):
+            return datetime.strptime(dt.replace("24:00", "00:00"), "%Y-%m-%d %H:%M") + timedelta(days=1)
+        return datetime.strptime(dt, "%Y-%m-%d %H:%M")
+
+    def get_programs(self) -> None:
+        max_ndays = 5
+        if int(self.cfg["FETCH_LIMIT"]) > max_ndays:
+            log.warning(
+                """
+
+***********************************************************************
+
+%s는 당일포함 %d일치만 EPG를 제공하고 있습니다.
+
+***********************************************************************
+            """,
+                self.provider_name,
+                max_ndays,
+            )
+        data = []
+        for nd in range(min(int(self.cfg["FETCH_LIMIT"]), max_ndays)):
+            day = date.today() + timedelta(days=nd)
+            url = "https://www.spotvnow.co.kr/api/v3/program/" + day.strftime("%Y-%m-%d")
+            try:
+                data.extend(self.request(url))
+            except Exception:
+                log.exception("데이터 가져오는 중 에러:")
+                continue
+
+        # 날짜의 경계에서 발생할 수 있는 중복 제거
+        _data = []
+        for _d in data:
+            _d.pop("date", None)
+            if _d not in _data:
+                _data.append(_d)
+
+        for idx, _ch in enumerate(self.req_channels):
+            log.info("%03d/%03d %s", idx + 1, len(self.req_channels), _ch)
+            try:
+                _epgs = self.__epgs_of_channel(_ch.id, _data, _ch.svcid)
+            except AssertionError as e:
+                log.warning("%s: %s", e, _ch)
+            except Exception:
+                log.exception("프로그램 파싱 중 예외: %s", _ch)
+            else:
+                _ch.programs.extend(_epgs)
+
+    def __epgs_of_channel(self, channelid: str, data: dict, svcid: str) -> List[EPGProgram]:
+        programs = [x for x in data if x["channelId"] == svcid]
+        assert programs, "EPG 정보가 없거나 없는 채널입니다"
+
+        _epgs = []
+        for p in programs:
+            _epg = EPGProgram(channelid)
+            _epg.title = p["title"]
+            _epg.stime = self.__dt(p["startTime"])
+            # 끝나는 시간이 없으면 해당일 자정으로 강제
+            _epg.etime = self.__dt(p["endTime"]) or (_epg.stime.replace(hour=0, minute=0) + timedelta(days=1))
+            if _epg.stime == _epg.etime:
+                continue
+
+            if m := self.title_regex.match(_epg.title):
+                _epg.title = m.group(2)
+                subs = []
+                if prefix := m.group(1):
+                    subs.append(prefix)
+                if sub := m.group(3):
+                    subs += [sub.replace(")(", ", ").replace(") (", ", ")]
+                title_sub = ", ".join(subs)
+                if num := m.group(4):
+                    title_sub += f"-{num}"
+                if title_sub:
+                    _epg.title_sub = title_sub
+                _epg.ep_num = m.group(5)
+            if p["type"] == 300:
+                # 100: live, 200: 본방송
+                _epg.rebroadcast = True
+            _epgs.append(_epg)
+        return _epgs

epg2xml/providers/tving.py

@@ -0,0 +1,180 @@
+import logging
+from datetime import date, datetime, timedelta
+from itertools import islice
+from typing import List
+
+import requests
+
+from epg2xml.providers import EPGProgram, EPGProvider
+
+log = logging.getLogger(__name__.rsplit(".", maxsplit=1)[-1].upper())
+today = date.today()
+
+PRIORITY_IMG_CODE = ["CAIC2300", "CAIC1600", "CAIC0100", "CAIC0400"]
+G_CODE = {
+    "CPTG0100": 0,
+    "CPTG0200": 7,
+    "CPTG0300": 12,
+    "CPTG0400": 15,
+    "CPTG0500": 19,
+    "CMMG0100": 0,
+    "CMMG0200": 12,
+    "CMMG0300": 15,
+    "CMMG0400": 19,
+}
+
+
+class TVING(EPGProvider):
+    """EPGProvider for TVING
+
+    데이터: jsonapi
+    요청수: #channels/20 * #days * 24/3
+    특이사항:
+    - 최대 20채널 최대 3시간 허용
+    """
+
+    referer = "https://www.tving.com/schedule/main.do"
+    tps = 3.0
+
+    url = "https://api.tving.com/v2/media/schedules"
+    base_params = {
+        "pageNo": "1",
+        "pageSize": "20",  # maximum 20
+        "order": "chno",
+        "scope": "all",
+        "adult": "all",
+        "free": "all",
+        "broadDate": "20200608",
+        "broadcastDate": "20200608",
+        "startBroadTime": "030000",  # 최대 3시간 간격
+        "endBroadTime": "060000",
+        # "channelCode": "C06941,C07381,...",
+        "screenCode": "CSSD0100",
+        "networkCode": "CSND0900",
+        "osCode": "CSOD0900",
+        "teleCode": "CSCD0900",
+        "apiKey": "1e7952d0917d6aab1f0293a063697610",
+    }
+
+    def __params(self, **params) -> dict:
+        """returns url parameters for api requests with base ones"""
+        p = self.base_params.copy()
+        p.update(params)
+        return p
+
+    def __get(self, url: str, **kwargs) -> List[dict]:
+        params = self.__params(**kwargs.pop("params", {}))
+        _page = 1
+        _results = []
+        while True:
+            params["pageNo"] = str(_page)
+            _data = self.request(url=url, params=params, **kwargs)
+            if _data["header"]["status"] != 200:
+                raise requests.exceptions.RequestException
+            _results.extend(_data["body"]["result"])
+            if _data["body"]["has_more"] == "Y":
+                _page += 1
+            else:
+                break
+        return _results
+
+    def get_svc_channels(self) -> List[dict]:
+        def get_imgurl(_item: dict):
+            for _code in PRIORITY_IMG_CODE:
+                try:
+                    img_list = [x for x in _item["image"] if x["code"] == _code]
+                    if not img_list:
+                        continue
+                    return "https://image.tving.com" + (img_list[0].get("url") or img_list[0]["url2"])
+                except Exception:
+                    pass
+            return None
+
+        params = {
+            "broadDate": today.strftime("%Y%m%d"),
+            "broadcastDate": today.strftime("%Y%m%d"),
+            "startBroadTime": datetime.now().strftime("%H0000"),
+            "endBroadTime": (datetime.now() + timedelta(hours=3)).strftime("%H0000"),
+        }
+        return [
+            {
+                "Name": x["channel_name"]["ko"],
+                "Icon_url": get_imgurl(x),
+                "ServiceId": x["channel_code"],
+                "Category": x["schedules"][0]["channel"]["category_name"]["ko"],
+            }
+            for x in self.__get(self.url, params=params)
+            if x["schedules"] is not None
+        ]
+
+    def get_programs(self) -> None:
+        def grouper(iterable, n):
+            it = iter(iterable)
+            group = tuple(islice(it, n))
+            while group:
+                yield group
+                group = tuple(islice(it, n))
+
+        for gid, chgroup in enumerate(grouper(self.req_channels, 20)):
+            schdict = {}
+            params = {"channelCode": ",".join([x.svcid.strip() for x in chgroup])}
+            for nd in range(int(self.cfg["FETCH_LIMIT"])):
+                day = today + timedelta(days=nd)
+                params.update({"broadDate": day.strftime("%Y%m%d"), "broadcastDate": day.strftime("%Y%m%d")})
+                for t in range(8):
+                    params.update({"startBroadTime": f"{t*3:02d}0000", "endBroadTime": f"{t*3+3:02d}0000"})
+                    for ch in self.__get(self.url, params=params):
+                        chcode = ch["channel_code"]
+                        schdict.setdefault(chcode, [])
+                        toappend = ch.get("schedules") or []
+                        try:
+                            # 3시간 단위로 요청된 스케줄 앞 뒤로 중복이 있을 수 있다.
+                            if schdict[chcode][-1] == toappend[0]:
+                                toappend = toappend[1:]
+                        except Exception:
+                            pass
+                        schdict[chcode] += toappend
+
+            for idx, _ch in enumerate(chgroup):
+                log.info("%03d/%03d %s", gid * 20 + idx + 1, len(self.req_channels), _ch)
+                try:
+                    _epgs = self.__epgs_of_channel(_ch.id, schdict[_ch.svcid])
+                except Exception:
+                    log.exception("프로그램 파싱 중 예외: %s", _ch)
+                else:
+                    _ch.programs.extend(_epgs)
+
+    def __epgs_of_channel(self, channelid: str, schedules: List[dict]) -> List[EPGProgram]:
+        _epgs = []
+        for sch in schedules:
+            _epg = EPGProgram(channelid)
+            # 공통
+            _epg.stime = datetime.strptime(str(sch["broadcast_start_time"]), "%Y%m%d%H%M%S")
+            _epg.etime = datetime.strptime(str(sch["broadcast_end_time"]), "%Y%m%d%H%M%S")
+            _epg.rebroadcast = sch["rerun_yn"] == "Y"
+
+            get_from = "movie" if sch["movie"] else "program"
+            img_code = "CAIM2100" if sch["movie"] else "CAIP0900"
+
+            _epg.rating = G_CODE[sch[get_from].get("grade_code", "CPTG0100")]
+            _epg.title = sch[get_from]["name"]["ko"]
+            _epg.title_sub = sch[get_from]["name"].get("en")
+            if cate1 := sch[get_from]["category1_name"].get("ko"):
+                _epg.categories = [cate1]
+            if cate2 := sch[get_from]["category2_name"].get("ko"):
+                _epg.categories = (_epg.categories or []) + [cate2]
+            _epg.cast = [{"name": x, "title": "actor"} for x in sch[get_from]["actor"]]
+            _epg.crew = [{"name": x, "title": "director"} for x in sch[get_from]["director"]]
+
+            poster = [x["url"] for x in sch[get_from]["image"] if x["code"] == img_code]
+            if poster:
+                _epg.poster_url = "https://image.tving.com" + poster[0]
+                # _prog.poster_url += '/dims/resize/236'
+
+            _epg.desc = sch[get_from]["story" if sch["movie"] else "synopsis"]["ko"]
+            if sch["episode"]:
+                episode = sch["episode"]["frequency"]
+                _epg.ep_num = "" if episode == 0 else str(episode)
+                _epg.desc = sch["episode"]["synopsis"]["ko"]
+            _epgs.append(_epg)
+        return _epgs

epg2xml/providers/wavve.py

@@ -0,0 +1,155 @@
+import logging
+from datetime import date, datetime, timedelta
+from functools import lru_cache
+from typing import List
+from xml.sax.saxutils import unescape
+
+from epg2xml.providers import EPGProgram, EPGProvider
+
+log = logging.getLogger(__name__.rsplit(".", maxsplit=1)[-1].upper())
+today = date.today()
+
+
+class WAVVE(EPGProvider):
+    """EPGProvider for WAVVE
+
+    데이터: jsonapi
+    요청수: 1
+    특이사항:
+    - 해외나 VPS는 차단 가능성이 높음
+    """
+
+    referer = "https://www.wavve.com/"
+    title_regex = r"^(.*?)(?:\s*[\(<]?([\d]+)회[\)>]?)?(?:\([월화수목금토일]?\))?(\([선별전주\(\)재방]*?재[\d방]?\))?\s*(?:\[(.+)\])?$"
+
+    base_url = "https://apis.wavve.com"
+    base_params = {
+        "apikey": "E5F3E0D30947AA5440556471321BB6D9",
+        "client_version": "6.0.1",
+        "device": "pc",
+        "drm": "wm",
+        "partner": "pooq",
+        "pooqzone": "none",
+        "region": "kor",
+        "targetage": "all",
+    }
+
+    def __init__(self, cfg):
+        super().__init__(cfg)
+        self.sess.headers.update({"wavve-credential": "none"})
+
+    def __url(self, url: str) -> str:
+        """completes partial urls from api response or for api request"""
+        if url.startswith(("http://", "https://")):
+            return url
+        if url.startswith("/"):
+            return self.base_url + url
+        return "https://" + url
+
+    def __params(self, **params) -> dict:
+        """returns url parameters for api requests with base ones"""
+        p = self.base_params.copy()
+        p.update(params)
+        return p
+
+    def __get(self, url: str, **kwargs):
+        url = self.__url(url)
+        params = self.__params(**kwargs.pop("params", {}))
+        return self.request(url, params=params, **kwargs)
+
+    def get_svc_channels(self) -> List[dict]:
+        today_str = today.strftime("%Y-%m-%d")
+        hour_min = datetime.now().hour // 3
+        # 현재 시간과 가까운 미래에 서비스 가능한 채널만 가져옴
+        params = {
+            "enddatetime": f"{today_str} {(hour_min+1)*3:02d}:00",
+            "genre": "all",
+            "limit": 500,
+            "offset": 0,
+            "startdatetime": f"{today_str} {hour_min*3:02d}:00",
+        }
+        return [
+            {
+                "Name": x["channelname"],
+                "Icon_url": self.__url(x["channelimage"]),
+                "ServiceId": x["channelid"],
+            }
+            for x in self.__get("/live/epgs", params=params)["list"]
+        ]
+
+    def __epg_of_program(self, channelid: str, data: dict) -> EPGProgram:
+        _epg = EPGProgram(channelid)
+        _epg.stime = datetime.strptime(data["starttime"], "%Y-%m-%d %H:%M")
+        _epg.etime = datetime.strptime(data["endtime"], "%Y-%m-%d %H:%M")
+        # 채널이름은 그대로 들어오고 프로그램 제목은 escape되어 들어옴
+        _epg.title = unescape(data["title"])
+        if m := self.title_regex.match(_epg.title):
+            _epg.title = m.group(1)
+            _epg.title_sub = m.group(4)
+            episode = (m.group(2) or "").replace("회", "").strip()
+            _epg.ep_num = None if episode == "0" else episode
+            _epg.rebroadcast = bool(m.group(3))
+        _epg.rating = 0 if data["targetage"] == "n" else int(data["targetage"])
+
+        # 추가 정보 가져오기
+        if not self.cfg["GET_MORE_DETAILS"]:
+            return _epg
+        programid = data["programid"].strip()
+        if not programid:
+            # 개별 programid가 없는 경우도 있으니 체크해야함
+            return _epg
+        detail = self.get_program_details(programid)
+        if not detail:
+            return _epg
+        # 여러가지 추가 정보가 제공되지만
+        # 방송되지 않은 미래의 프로그램/에피소드 정보는 반영되지 않았기에
+        # 일부 정보만 유효함을 유념
+        synopsis = detail["seasonsynopsis"] or detail["programsynopsis"] or detail["episodesynopsis"]
+        _epg.desc = "\n".join(
+            [x.replace("<br>", "\n").strip() for x in synopsis.splitlines()]
+        )  # carriage return(\r) 제거, <br> 제거
+        _epg.categories = [detail["genretext"].strip()]
+        _epg.poster_url = self.__url(detail["seasonposterimage"].strip())
+        _epg.keywords = [x["text"] for x in detail["tags"]["list"]]
+        actors = detail.get("season_actors") or detail.get("actors") or {"list": []}
+        directors = detail.get("season_directors") or detail.get("directors") or {"list": []}
+        writers = detail.get("season_writers") or detail.get("writers") or {"list": []}
+        _epg.cast = [{"name": x["text"], "title": "actor"} for x in actors["list"]]
+        _epg.crew = [{"name": x["text"], "title": "director"} for x in directors["list"]]
+        _epg.crew += [{"name": x["text"], "title": "writer"} for x in writers["list"]]
+        return _epg
+
+    def get_programs(self) -> None:
+        # parameters for requests
+        params = {
+            "enddatetime": (today + timedelta(days=int(self.cfg["FETCH_LIMIT"]) - 1)).strftime("%Y-%m-%d 24:00"),
+            "genre": "all",
+            "limit": 500,
+            "offset": 0,
+            "startdatetime": today.strftime("%Y-%m-%d 00:00"),
+        }
+        channeldict = {x["channelid"]: x for x in self.__get("/live/epgs", params=params)["list"]}
+
+        for idx, _ch in enumerate(self.req_channels):
+            log.info("%03d/%03d %s", idx + 1, len(self.req_channels), _ch)
+            for program in channeldict[_ch.svcid]["list"]:
+                try:
+                    _epg = self.__epg_of_program(_ch.id, program)
+                except Exception:
+                    log.exception("프로그램 파싱 중 예외: %s", _ch)
+                else:
+                    _ch.programs.append(_epg)
+
+    @lru_cache
+    def get_program_details(self, programid: str) -> dict:
+        try:
+            params = {"history": "season", "programid": programid}
+            data = self.__get("/fz/vod/programs/landing", params=params)
+            if data.get("resultcode") in ["550"]:
+                # 애초에 유효하지 않은 programid가 있을 수 있음
+                # { "resultcode": "550", "resultmessage": "해당 데이터가 없습니다." }
+                return None
+            return self.__get(f"/fz/vod/contents-detail/{data['content_id'].strip()}")
+        except Exception:
+            log.exception("프로그램 상세 정보 요청 중 예외: %s", programid)
+            return None

epg2xml/utils.py

@@ -0,0 +1,196 @@
+import json
+import logging
+import re
+import sys
+import threading
+import time
+import xml.etree.ElementTree as ET
+from functools import wraps
+from math import floor
+from typing import Callable
+
+from bs4 import BeautifulSoup, FeatureNotFound
+
+log = logging.getLogger("UTILS")
+
+
+def dump_json(file_path, data) -> int:
+    with open(file_path, "w", encoding="utf-8") as f:
+        txt = json.dumps(data, ensure_ascii=False, indent=2)
+        # for compact form of channellist in json files
+        txt = re.sub(r",\n\s{8}\"", ', "', txt)
+        txt = re.sub(r"\s{6}{\s+(.*)\s+}", r"      { \g<1> }", txt)
+        return f.write(txt)
+
+
+# https://stackoverflow.com/a/22273639
+_illegal_unichrs = [
+    (0x00, 0x08),
+    (0x0B, 0x0C),
+    (0x0E, 0x1F),
+    (0x7F, 0x84),
+    (0x86, 0x9F),
+    (0xFDD0, 0xFDDF),
+    (0xFFFE, 0xFFFF),
+]
+if sys.maxunicode >= 0x10000:  # not narrow build
+    _illegal_unichrs.extend(
+        [
+            (0x1FFFE, 0x1FFFF),
+            (0x2FFFE, 0x2FFFF),
+            (0x3FFFE, 0x3FFFF),
+            (0x4FFFE, 0x4FFFF),
+            (0x5FFFE, 0x5FFFF),
+            (0x6FFFE, 0x6FFFF),
+            (0x7FFFE, 0x7FFFF),
+            (0x8FFFE, 0x8FFFF),
+            (0x9FFFE, 0x9FFFF),
+            (0xAFFFE, 0xAFFFF),
+            (0xBFFFE, 0xBFFFF),
+            (0xCFFFE, 0xCFFFF),
+            (0xDFFFE, 0xDFFFF),
+            (0xEFFFE, 0xEFFFF),
+            (0xFFFFE, 0xFFFFF),
+            (0x10FFFE, 0x10FFFF),
+        ]
+    )
+_illegal_ranges = [rf"{chr(low)}-{chr(high)}" for (low, high) in _illegal_unichrs]
+_illegal_xml_chars_RE = re.compile("[" + "".join(_illegal_ranges) + "]")
+
+
+class Element(ET.Element):
+    def __init__(self, *args, **kwargs):
+        attrib = kwargs.pop("attrib", {})
+        super().__init__(args[0], attrib=attrib, **kwargs)
+        if len(args) > 1:
+            self.text = args[1]
+
+    def indent(self, space="  ", level=0):
+        if level < 0:
+            raise ValueError(f"Initial indentation level must be >= 0, got {level}")
+        if len(self) == 0:
+            return
+
+        # Reduce the memory consumption by reusing indentation strings.
+        indentations = ["\n" + level * space]
+
+        def _indent_children(elem, level):
+            # Start a new indentation level for the first child.
+            child_level = level + 1
+            try:
+                child_indentation = indentations[child_level]
+            except IndexError:
+                child_indentation = indentations[level] + space
+                indentations.append(child_indentation)
+
+            if not elem.text or not elem.text.strip():
+                elem.text = child_indentation
+
+            for child in elem:
+                if len(child):
+                    _indent_children(child, child_level)
+                if not child.tail or not child.tail.strip():
+                    child.tail = child_indentation
+
+            # Dedent after the last child by overwriting the previous indentation.
+            if not child.tail.strip():  # pylint: disable=undefined-loop-variable
+                child.tail = indentations[level]  # pylint: disable=undefined-loop-variable
+
+        _indent_children(self, 0)
+
+    def tostring(self, space="  ", level=0):
+        self.indent(space=space, level=level)
+        return _illegal_xml_chars_RE.sub("", space * level + ET.tostring(self, encoding="unicode"))
+
+
+class PrefixLogger(logging.LoggerAdapter):
+    def __init__(self, logger, prefix):
+        super().__init__(logger, {})
+        self.prefix = prefix
+
+    def process(self, msg, kwargs):
+        return f"{self.prefix} {msg}", kwargs
+
+
+class ParserBeautifulSoup(BeautifulSoup):
+    """A ``bs4.BeautifulSoup`` that picks the first available parser."""
+
+    def insert_before(self, *args):
+        pass
+
+    def insert_after(self, *args):
+        pass
+
+    def __init__(self, markup, **kwargs):
+        # pick the first parser available
+        for parser in ["lxml", "html.parser"]:
+            try:
+                super().__init__(markup, parser, **kwargs)
+                return
+            except FeatureNotFound:
+                pass
+
+        raise FeatureNotFound
+
+
+class RateLimiter:
+    """original implementation by tomasbasham/ratelimit"""
+
+    try:
+        now: Callable = time.monotonic  # Use monotonic time if available
+    except AttributeError:
+        now: Callable = time.time  # otherwise fall back to the system clock
+
+    def __init__(self, calls: int = 15, period: float = 900.0, tps: float = None):
+        if tps is not None:
+            if tps <= 0.0:
+                raise ValueError("tps must be positive")
+            calls, period = 1, 1 / tps
+        self.max_calls = max(1, min(sys.maxsize, floor(calls)))
+        self.period = period
+
+        # Initialise the decorator state.
+        self.last_reset = self.now()
+        self.num_calls = 0
+
+        # Add thread safety.
+        self.lock = threading.RLock()
+
+    def __call__(self, func: Callable) -> Callable:
+        """
+        Return a wrapped function that prevents further function invocations if
+        previously called within a specified period of time.
+        """
+
+        @wraps(func)
+        def wrapper(*args, **kargs):
+            """
+            Extend the behaviour of the decorated function, forwarding function
+            invocations previously called no sooner than a specified period of
+            time. The decorator will raise an exception if the function cannot
+            be called so the caller may implement a retry strategy such as an
+            exponential backoff.
+            """
+            with self.lock:
+                period_remaining = self.__period_remaining()
+
+                # If the time window has elapsed then reset.
+                if period_remaining <= 0:
+                    self.num_calls = 0
+                    self.last_reset = self.now()
+
+                # Increase the number of attempts to call the function.
+                self.num_calls += 1
+
+                # If the number of attempts to call the function exceeds the maximum
+                if self.num_calls > self.max_calls:
+                    self.last_reset = self.now() + period_remaining  # for future call
+                    time.sleep(period_remaining)
+                    return func(*args, **kargs)
+            return func(*args, **kargs)
+
+        return wrapper
+
+    def __period_remaining(self) -> float:
+        elapsed = self.now() - self.last_reset
+        return self.period - elapsed