commit ac281878ced6080a0c2e700c2da627a569032cce Author: jvved Date: Thu Jan 30 18:17:43 2025 -0500 first commit diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..26d3352 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,3 @@ +# Default ignored files +/shelf/ +/workspace.xml diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml new file mode 100644 index 0000000..105ce2d --- /dev/null +++ b/.idea/inspectionProfiles/profiles_settings.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..d460eee --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,6 @@ + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..af20f60 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/testpks.iml b/.idea/testpks.iml new file mode 100644 index 0000000..5c0fbc8 --- /dev/null +++ b/.idea/testpks.iml @@ -0,0 +1,10 @@ + + + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.read_stats.py.swo b/.read_stats.py.swo new file mode 100644 index 0000000..12505a7 Binary files /dev/null and b/.read_stats.py.swo differ diff --git a/.read_stats.py.swp b/.read_stats.py.swp new file mode 100644 index 0000000..d016989 Binary files /dev/null and b/.read_stats.py.swp differ diff --git a/app1.ini b/app1.ini new file mode 100644 index 0000000..0b483bb --- /dev/null +++ b/app1.ini @@ -0,0 +1,13 @@ +[uwsgi] + +master=true +workers=3 +threads=true + +wsgi-file=foobar.py +vacuum=true +socket=127.0.0.1:9027 + +stats-server=run/9027_stats.sock + +subscribe2=addr=127.0.0.1:9027,key=app1.pikesquares.local,server=127.0.0.1:9700 diff --git a/app2.ini b/app2.ini new file mode 100644 index 0000000..e1ba574 --- /dev/null +++ b/app2.ini @@ -0,0 +1,12 @@ +[uwsgi] + +master = true +workers = 5 +threads = true + +wsgi-file = foobar.py +vacuum=true +socket=127.0.0.1:9028 +stats-server=run/app2_stats.sock + +subscribe2=addr=127.0.0.1:9028,key=app1.pikesquares.local,server=127.0.0.1:9700 diff --git a/app3.ini b/app3.ini new file mode 100644 index 0000000..6800610 --- /dev/null +++ b/app3.ini @@ -0,0 +1,12 @@ +[uwsgi] + +master = true +workers = 3 +threads = true + +wsgi-file = foobar.py +socket=127.0.0.1:9029 + +stats-server=run/app1_stats.sock + +subscribe2=addr=127.0.0.1:9027,key=app1.pikesquares.local,server=127.0.0.1:9700 diff --git a/appX.ini b/appX.ini new file mode 100644 index 0000000..23eae4e --- /dev/null +++ b/appX.ini @@ -0,0 +1,13 @@ +[uwsgi] + +master = true +workers = 3 +threads = true +vacuum=true + +wsgi-file = foobar.py +socket=127.0.0.1:9029 + +stats-server=run/appX_stats.sock + +subscribe2=addr=127.0.0.1:9028,key=appX.pikesquares.local,server=127.0.0.1:9701 diff --git a/app_stats.py b/app_stats.py new file mode 100644 index 0000000..e49edf4 --- /dev/null +++ b/app_stats.py @@ -0,0 +1,28 @@ +from random import randint +from stats import AppStats +from pathlib import Path + + +def read_stats(socket_address): + sockets = { + "9027_stats.sock": "{app1 - stats}" + } + if socket_address in sockets: + return sockets[socket_address] + else: + return "could not lookup socket" + +if __name__ == "__main__": + counter = 0 + nodes =["127.0.0.1:9027", "127.0.0.1:9030"] + for node in nodes: + port = node.split(":")[-1] + socket_address = f"{port}_stats.sock" + stats = read_stats(socket_address) + print(stats) + + #print(f"{worker_count=}") + #counter = counter + worker_count + #print(counter) + + diff --git a/foobar.py b/foobar.py new file mode 100644 index 0000000..e64b8e7 --- /dev/null +++ b/foobar.py @@ -0,0 +1,3 @@ +def application(env, start_response): + start_response('200 OK', [('Content-Type','text/html')]) + return [b"Hello World"] \ No newline at end of file diff --git a/lessons.py b/lessons.py new file mode 100644 index 0000000..55360f3 --- /dev/null +++ b/lessons.py @@ -0,0 +1,16 @@ +class HelloWorld: + def __init__(self, num_iters): + self.num_iters = num_iters + self.counter = 0 + + + def __iter__(self): + qreturn self + qreturn self + + def __next__(self): + if self.counter < self.num_iters: + self.counter += 1 + return "Hello World" + raise StopIteration + diff --git a/read_stats.py b/read_stats.py new file mode 100644 index 0000000..4c27a1b --- /dev/null +++ b/read_stats.py @@ -0,0 +1,143 @@ +from pathlib import Path +from stats import AppStats +import errno +import json +import traceback +import socket +import stats +from stats import WorkerAppStats +from stats import AppStats + +from rich.console import Console + +from stats import RouterSubscription +import time +from rich.live import Live +from rich.table import Table +import pydantic + +class RouterStats(pydantic.BaseModel): + model_config = pydantic.ConfigDict(strict=True) + version: str + pid: int = pydantic.Field(ge=0) + uid: int = pydantic.Field(ge=0) + gid: int = pydantic.Field(ge=0) + cwd: str + active_sessions: int = pydantic.Field(ge=0) + http: list[str] # ['0.0.0.0:8034', '127.0.0.1:5700'], + subscriptions: list[RouterSubscription] + cheap: int = pydantic.Field(ge=0) + + + + +def run(stats_address): + if not all([stats_address.exists(), stats_address.is_socket()]): + raise Exception(f"unable to read stats from {(stats_address)}") + + def unix_addr(arg): + sfamily = socket.AF_UNIX + addr = arg + return sfamily, addr, socket.gethostname() + + js = "" + sfamily, addr, host = unix_addr(stats_address) + # console.info(f"{sfamily=} {str(addr)=} {host=}") + + try: + s = None + s = socket.socket(sfamily, socket.SOCK_STREAM) + s.connect(str(addr)) + while True: + data = s.recv(4096) + if len(data) < 1: + break + js += data.decode('utf8', 'ignore') + if s: + s.close() + except ConnectionRefusedError as e: + print('connection refused') + except FileNotFoundError as e: + print(f"socket @ {addr} not available") + except IOError as e: + if e.errno != errno.EINTR: + #uwsgi.log(f"socket @ {addr} not available") + pass + except Exception: + print(traceback.format_exc()) + else: + try: + return json.loads(js) + except json.JSONDecodeError: + print(traceback.format_exc()) + print(js) + +if __name__ == "__main__": + stats_address = Path("run/sub1-stats.sock") + stats = run(stats_address) + + +class AppStats(pydantic.BaseModel): + model_config = pydantic.ConfigDict(strict=True) + version: str + listen_queue: int + listen_queue_errors: int + signal_queue: int + load: int + pid: int + uid: int + gid: int + cwd: str + locks: list[dict[str, int]] + sockets: list[stats.SocketStats] + workers: list[stats.WorkerStats] + + def read_stats(socket_address): + sockets = { + "9027_stats.sock": "{app1 - stats}" + } + if socket_address in sockets: + return sockets[socket_address] + else: + return "could not lookup socket" + + if __name__ == "__main__": + counter = 0 + nodes = ["127.0.0.1:9027", "127.0.0.1:9030"] + + + for node in nodes: + port = node.split(":")[-1] + socket_address = f"{port}_stats.sock" + stats = read_stats(socket_address) + print(stats) + + stats2 = run(Path("run/app1_stats.sock")) + app_stats = AppStats(**stats2) + print(app_stats.workers) + worker_amount = app_stats.workers + print(len(app_stats.workers)) + + + + + + router_stats = RouterStats(**stats) + print(router_stats) + + # import ipdb;ipdb.set_trace() + for sub in router_stats.subscriptions: + print(f"{sub.key=}") + + table = Table() + table.add_column("Virtual Hosts", justify="right", style="cyan", no_wrap=True) + table.add_column("Nodes", style="magenta") + table.add_column("Total Workers", style="magenta") + + table.add_row(sub.key, f"{len(sub.nodes)}") + + + + console = Console() + console.print(table) + diff --git a/read_stats2.py b/read_stats2.py new file mode 100644 index 0000000..bb28fb8 --- /dev/null +++ b/read_stats2.py @@ -0,0 +1,112 @@ +from pathlib import Path +import errno +import json +import traceback +import socket + +from rich.console import Console + +from stats import RouterSubscription +import time +from rich.live import Live +from rich.table import Table +import pydantic + +class RouterStats(pydantic.BaseModel): + model_config = pydantic.ConfigDict(strict=True) + version: str + pid: int = pydantic.Field(ge=0) + uid: int = pydantic.Field(ge=0) + gid: int = pydantic.Field(ge=0) + cwd: str + active_sessions: int = pydantic.Field(ge=0) + http: list[str] # ['0.0.0.0:8034', '127.0.0.1:5700'], + subscriptions: list[RouterSubscription] + cheap: int = pydantic.Field(ge=0) + + + + +def run(stats_address): + if not all([stats_address.exists(), stats_address.is_socket()]): + raise Exception(f"unable to read stats from {(stats_address)}") + + def unix_addr(arg): + sfamily = socket.AF_UNIX + addr = arg + return sfamily, addr, socket.gethostname() + + js = "" + sfamily, addr, host = unix_addr(stats_address) + # console.info(f"{sfamily=} {str(addr)=} {host=}") + + try: + s = None + s = socket.socket(sfamily, socket.SOCK_STREAM) + s.connect(str(addr)) + while True: + data = s.recv(4096) + if len(data) < 1: + break + js += data.decode('utf8', 'ignore') + if s: + s.close() + except ConnectionRefusedError as e: + print('connection refused') + except FileNotFoundError as e: + print(f"socket @ {addr} not available") + except IOError as e: + if e.errno != errno.EINTR: + #uwsgi.log(f"socket @ {addr} not available") + pass + except Exception: + print(traceback.format_exc()) + else: + try: + return json.loads(js) + except json.JSONDecodeError: + print(traceback.format_exc()) + print(js) + +if __name__ == "__main__": + stats_address = Path("run/sub2-stats.sock") + stats = run(stats_address) + print(stats) + router_stats = RouterStats(**stats) + + + #import ipdb; ipdb.set_trace() + table = Table() + table.add_column("Name", justify="right", style="cyan", no_wrap=True) + table.add_column("Subscription", style="magenta") + table.add_column("Worker's count", justify="right", style="green") + + for sub in router_stats.subscriptions: + table.add_row("key", f"{sub.key=}", f"{len(sub.nodes)=}") + table.add_row("hash", f"{sub.hash=}") + table.add_row("hits", f"{sub.hits=}") + table.add_row("sni_enabled", f"{sub.sni_enabled=}") + #print(f"{sub.key=}") + for n in sub.nodes: + table.add_row("node's name", f"{n.name=}") + # table.add_row("node2", f"{n.name=}") + + + console = Console() + console.print(table) + + + + + # for field, value in router_stats.__dict__.items(): + # table.add_row(field, str(value)) + # + # + # + # console = Console() + # console.print(table) + # + # for i in router_stats.__dict__.items(): + # print(i) + + diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..e9f9419 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +ipdb +uwsgi +rich +pydantic diff --git a/stats.py b/stats.py new file mode 100644 index 0000000..1499e6b --- /dev/null +++ b/stats.py @@ -0,0 +1,207 @@ +from pathlib import Path + +import pydantic + + +class VirtualHost(pydantic.BaseModel): + address: str + certificate_path: Path + certificate_key: Path + server_names: list[str] + protocol: str = "https" + static_files_mapping: dict = {} + + @property + def is_https(self): + return all([ + self.certificate_key, + self.certificate_path + ]) + + +class Router(pydantic.BaseModel): + router_id: str + subscription_server_address: str + app_name: str + + @pydantic.computed_field + def subscription_server_port(self) -> int: + try: + return int(self.subscription_server_address.split(":")[-1]) + except IndexError: + return 0 + + @pydantic.computed_field + def subscription_server_key(self) -> str: + # return f"{self.app_name}.pikesquares.dev:{self.subscription_server_port}" + print("subscription_server_key") + return f"{self.app_name}.pikesquares.dev" + + @pydantic.computed_field + def subscription_server_protocol(self) -> str: + return "http" if str(self.subscription_server_port).startswith("9") else "https" + + +class WsgiAppOptions(pydantic.BaseModel): + root_dir: Path + pyvenv_dir: Path + wsgi_file: Path + wsgi_module: str + routers: list[Router] = [] + project_id: str + workers: int = 3 + + +class RouterNode(pydantic.BaseModel): + model_config = pydantic.ConfigDict(strict=True) + name: str + modifier1: int + modifier2: int + last_check: int + pid: int + uid: int + gid: int + requests: int + last_requests: int + tx: int + rx: int + cores: int + load: int + weight: int + wrr: int + ref: int + failcnt: int + death_mark: int + + +class RouterSubscription(pydantic.BaseModel): + model_config = pydantic.ConfigDict(strict=True) + key: str # 'muffled-castle.pikesquares.dev:5700' + hash: int + hits: int = pydantic.Field(ge=0) + sni_enabled: int + nodes: list[RouterNode] + + +class RouterStats(pydantic.BaseModel): + model_config = pydantic.ConfigDict(strict=True) + version: str + pid: int = pydantic.Field(ge=0) + uid: int = pydantic.Field(ge=0) + gid: int = pydantic.Field(ge=0) + cwd: str + active_sessions: int = pydantic.Field(ge=0) + http: list[str] # ['0.0.0.0:8034', '127.0.0.1:5700'], + subscriptions: list[RouterSubscription] + cheap: int = pydantic.Field(ge=0) + + +class DeviceAppStats(pydantic.BaseModel): + model_config = pydantic.ConfigDict(strict=True) + id: str # "project_sandbox.json", + pid: int + born: int + last_mod: int + last_heartbeat: int + loyal: int + ready: int + accepting: int + last_loyal: int + last_ready: int + last_accepting: int + first_run: int + last_run: int + cursed: int + zerg: int + on_demand: str + uid: int + gid: int + monitor: str + respawns: int + + +class DeviceStats(pydantic.BaseModel): + model_config = pydantic.ConfigDict(strict=True) + version: str + pid: int = pydantic.Field(ge=0) + uid: int = pydantic.Field(ge=0) + gid: int = pydantic.Field(ge=0) + cwd: str + emperor: list[str] + emperor_tyrant: int + throttle_level: int + vassals: list[DeviceAppStats] + blacklist: list + + +class SocketStats(pydantic.BaseModel): + model_config = pydantic.ConfigDict(strict=True) + name: str # "127.0.0.1:4017" + proto: str # "uwsgi" + queue: int + max_queue: int + shared: int + can_offload: int + + +class WorkerAppStats(pydantic.BaseModel): + model_config = pydantic.ConfigDict(strict=True) + id: int + modifier1: int + mountpoint: str + startup_time: int + requests: int + exceptions: int + chdir: str + + +class WorkerCoresStats(pydantic.BaseModel): + model_config = pydantic.ConfigDict(strict=True) + id: int + requests: int + static_requests: int + routed_requests: int + offloaded_requests: int + write_errors: int + read_errors: int + in_request: int + vars: list + req_info: dict + + +class WorkerStats(pydantic.BaseModel): + model_config = pydantic.ConfigDict(strict=True) + id: int + pid: int + accepting: int + requests: int + delta_requests: int + exceptions: int + harakiri_count: int + signals: int + signal_queue: int + status: str # "idle", + rss: int + vsz: int + running_time: int + last_spawn: int + respawn_count: int + tx: int + avg_rt: int + apps: list[WorkerAppStats] + + +class AppStats(pydantic.BaseModel): + model_config = pydantic.ConfigDict(strict=True) + version: str + listen_queue: int + listen_queue_errors: int + signal_queue: int + load: int + pid: int + uid: int + gid: int + cwd: str + locks: list[dict[str, int]] + sockets: list[SocketStats] + workers: list[WorkerStats] diff --git a/sub1.ini b/sub1.ini new file mode 100644 index 0000000..8c1ce56 --- /dev/null +++ b/sub1.ini @@ -0,0 +1,8 @@ +[uwsgi] +strict=true +master=true +touch-reload=sub1.ini +vacuum=true +http=0.0.0.0:8934 +http-subscription-server=127.0.0.1:9700 +http-stats=run/sub1-stats.sock diff --git a/sub2.ini b/sub2.ini new file mode 100644 index 0000000..2b6a56c --- /dev/null +++ b/sub2.ini @@ -0,0 +1,8 @@ +[uwsgi] +strict=true +master=true +touch-reload=sub2.ini +vacuum=true +http=0.0.0.0:8935 +http-subscription-server=127.0.0.1:9701 +http-stats=run/sub2-stats.sock diff --git a/text.txt b/text.txt new file mode 100644 index 0000000..715374c --- /dev/null +++ b/text.txt @@ -0,0 +1,4 @@ +My name is Julia. +I live in NYC. +I study Python. +In my spare time I go to gym, run and watch movies on projector. \ No newline at end of file