first commit

.idea/.gitignore

@@ -0,0 +1,3 @@
+# Default ignored files
+/shelf/
+/workspace.xml

.idea/inspectionProfiles/profiles_settings.xml

@@ -0,0 +1,6 @@
+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>

.idea/misc.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="Black">
+    <option name="sdkName" value="Python 3.12 (testpks)" />
+  </component>
+</project>

.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/testpks.iml" filepath="$PROJECT_DIR$/.idea/testpks.iml" />
+    </modules>
+  </component>
+</project>

.idea/testpks.iml

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$">
+      <excludeFolder url="file://$MODULE_DIR$/.venv" />
+    </content>
+    <orderEntry type="jdk" jdkName="Python 3.12 (testpks)" jdkType="Python SDK" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>

.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+  </component>
+</project>

app1.ini

@@ -0,0 +1,13 @@
+[uwsgi]
+
+master=true
+workers=3
+threads=true
+
+wsgi-file=foobar.py
+vacuum=true
+socket=127.0.0.1:9027
+
+stats-server=run/9027_stats.sock
+
+subscribe2=addr=127.0.0.1:9027,key=app1.pikesquares.local,server=127.0.0.1:9700

app2.ini

@@ -0,0 +1,12 @@
+[uwsgi]
+
+master = true
+workers = 5
+threads = true
+
+wsgi-file = foobar.py
+vacuum=true
+socket=127.0.0.1:9028
+stats-server=run/app2_stats.sock
+
+subscribe2=addr=127.0.0.1:9028,key=app1.pikesquares.local,server=127.0.0.1:9700

app3.ini

@@ -0,0 +1,12 @@
+[uwsgi]
+
+master = true
+workers = 3
+threads = true
+
+wsgi-file = foobar.py
+socket=127.0.0.1:9029
+
+stats-server=run/app1_stats.sock
+
+subscribe2=addr=127.0.0.1:9027,key=app1.pikesquares.local,server=127.0.0.1:9700

appX.ini

@@ -0,0 +1,13 @@
+[uwsgi]
+
+master = true
+workers = 3
+threads = true
+vacuum=true
+
+wsgi-file = foobar.py
+socket=127.0.0.1:9029
+
+stats-server=run/appX_stats.sock
+
+subscribe2=addr=127.0.0.1:9028,key=appX.pikesquares.local,server=127.0.0.1:9701

app_stats.py

@@ -0,0 +1,28 @@
+from random import randint
+from stats import AppStats
+from pathlib import Path
+
+
+def read_stats(socket_address):
+    sockets = {
+        "9027_stats.sock": "{app1 - stats}"
+    }
+    if socket_address in sockets:
+        return sockets[socket_address]
+    else:
+        return "could not lookup socket"
+
+if __name__ == "__main__":
+    counter = 0
+    nodes =["127.0.0.1:9027", "127.0.0.1:9030"]
+    for node in nodes:
+        port = node.split(":")[-1]
+        socket_address = f"{port}_stats.sock"
+        stats = read_stats(socket_address)
+        print(stats)
+
+        #print(f"{worker_count=}")
+        #counter = counter + worker_count
+    #print(counter)
+
+

foobar.py

@@ -0,0 +1,3 @@
+def application(env, start_response):
+    start_response('200 OK', [('Content-Type','text/html')])
+    return [b"Hello World"]

lessons.py

@@ -0,0 +1,16 @@
+class HelloWorld:
+    def __init__(self, num_iters):
+        self.num_iters = num_iters
+        self.counter = 0
+
+
+    def __iter__(self):
+       qreturn self
+       qreturn self
+
+    def __next__(self):
+        if self.counter < self.num_iters:
+            self.counter += 1
+            return "Hello World"
+    raise StopIteration
+

read_stats.py

@@ -0,0 +1,143 @@
+from pathlib import Path
+from stats import AppStats
+import errno
+import json
+import traceback
+import socket
+import stats
+from stats import WorkerAppStats
+from stats import AppStats
+
+from rich.console import Console
+
+from stats import RouterSubscription
+import time
+from rich.live import Live
+from rich.table import Table
+import pydantic
+
+class RouterStats(pydantic.BaseModel):
+    model_config = pydantic.ConfigDict(strict=True)
+    version: str
+    pid: int = pydantic.Field(ge=0)
+    uid: int = pydantic.Field(ge=0)
+    gid: int = pydantic.Field(ge=0)
+    cwd: str
+    active_sessions: int = pydantic.Field(ge=0)
+    http: list[str]  # ['0.0.0.0:8034', '127.0.0.1:5700'],
+    subscriptions: list[RouterSubscription]
+    cheap: int = pydantic.Field(ge=0)
+
+
+
+
+def run(stats_address):
+    if not all([stats_address.exists(), stats_address.is_socket()]):
+        raise Exception(f"unable to read stats from {(stats_address)}")
+
+    def unix_addr(arg):
+        sfamily = socket.AF_UNIX
+        addr = arg
+        return sfamily, addr, socket.gethostname()
+
+    js = ""
+    sfamily, addr, host = unix_addr(stats_address)
+    # console.info(f"{sfamily=} {str(addr)=} {host=}")
+
+    try:
+        s = None
+        s = socket.socket(sfamily, socket.SOCK_STREAM)
+        s.connect(str(addr))
+        while True:
+            data = s.recv(4096)
+            if len(data) < 1:
+                break
+            js += data.decode('utf8', 'ignore')
+        if s:
+            s.close()
+    except ConnectionRefusedError as e:
+        print('connection refused')
+    except FileNotFoundError as e:
+        print(f"socket @ {addr} not available")
+    except IOError as e:
+        if e.errno != errno.EINTR:
+            #uwsgi.log(f"socket @ {addr} not available")
+            pass
+    except Exception:
+        print(traceback.format_exc())
+    else:
+        try:
+            return json.loads(js)
+        except json.JSONDecodeError:
+            print(traceback.format_exc())
+            print(js)
+
+if __name__ == "__main__":
+    stats_address = Path("run/sub1-stats.sock")
+    stats = run(stats_address)
+
+
+class AppStats(pydantic.BaseModel):
+    model_config = pydantic.ConfigDict(strict=True)
+    version: str
+    listen_queue: int
+    listen_queue_errors: int
+    signal_queue: int
+    load: int
+    pid: int
+    uid: int
+    gid: int
+    cwd: str
+    locks: list[dict[str, int]]
+    sockets: list[stats.SocketStats]
+    workers: list[stats.WorkerStats]
+
+    def read_stats(socket_address):
+        sockets = {
+            "9027_stats.sock": "{app1 - stats}"
+        }
+        if socket_address in sockets:
+            return sockets[socket_address]
+        else:
+            return "could not lookup socket"
+
+    if __name__ == "__main__":
+        counter = 0
+        nodes = ["127.0.0.1:9027", "127.0.0.1:9030"]
+
+
+        for node in nodes:
+            port = node.split(":")[-1]
+            socket_address = f"{port}_stats.sock"
+            stats = read_stats(socket_address)
+            print(stats)
+
+            stats2 = run(Path("run/app1_stats.sock"))
+            app_stats = AppStats(**stats2)
+            print(app_stats.workers)
+            worker_amount = app_stats.workers
+            print(len(app_stats.workers))
+
+
+
+
+
+    router_stats = RouterStats(**stats)
+    print(router_stats)
+
+    # import ipdb;ipdb.set_trace()
+    for sub in router_stats.subscriptions:
+        print(f"{sub.key=}")
+
+    table = Table()
+    table.add_column("Virtual Hosts", justify="right", style="cyan", no_wrap=True)
+    table.add_column("Nodes", style="magenta")
+    table.add_column("Total Workers", style="magenta")
+
+    table.add_row(sub.key, f"{len(sub.nodes)}")
+
+
+
+    console = Console()
+    console.print(table)
+

read_stats2.py

@@ -0,0 +1,112 @@
+from pathlib import Path
+import errno
+import json
+import traceback
+import socket
+
+from rich.console import Console
+
+from stats import RouterSubscription
+import time
+from rich.live import Live
+from rich.table import Table
+import pydantic
+
+class RouterStats(pydantic.BaseModel):
+    model_config = pydantic.ConfigDict(strict=True)
+    version: str
+    pid: int = pydantic.Field(ge=0)
+    uid: int = pydantic.Field(ge=0)
+    gid: int = pydantic.Field(ge=0)
+    cwd: str
+    active_sessions: int = pydantic.Field(ge=0)
+    http: list[str]  # ['0.0.0.0:8034', '127.0.0.1:5700'],
+    subscriptions: list[RouterSubscription]
+    cheap: int = pydantic.Field(ge=0)
+
+
+
+
+def run(stats_address):
+    if not all([stats_address.exists(), stats_address.is_socket()]):
+        raise Exception(f"unable to read stats from {(stats_address)}")
+
+    def unix_addr(arg):
+        sfamily = socket.AF_UNIX
+        addr = arg
+        return sfamily, addr, socket.gethostname()
+
+    js = ""
+    sfamily, addr, host = unix_addr(stats_address)
+    # console.info(f"{sfamily=} {str(addr)=} {host=}")
+
+    try:
+        s = None
+        s = socket.socket(sfamily, socket.SOCK_STREAM)
+        s.connect(str(addr))
+        while True:
+            data = s.recv(4096)
+            if len(data) < 1:
+                break
+            js += data.decode('utf8', 'ignore')
+        if s:
+            s.close()
+    except ConnectionRefusedError as e:
+        print('connection refused')
+    except FileNotFoundError as e:
+        print(f"socket @ {addr} not available")
+    except IOError as e:
+        if e.errno != errno.EINTR:
+            #uwsgi.log(f"socket @ {addr} not available")
+            pass
+    except Exception:
+        print(traceback.format_exc())
+    else:
+        try:
+            return json.loads(js)
+        except json.JSONDecodeError:
+            print(traceback.format_exc())
+            print(js)
+
+if __name__ == "__main__":
+    stats_address = Path("run/sub2-stats.sock")
+    stats = run(stats_address)
+    print(stats)
+    router_stats = RouterStats(**stats)
+
+
+    #import ipdb; ipdb.set_trace()
+    table = Table()
+    table.add_column("Name", justify="right", style="cyan", no_wrap=True)
+    table.add_column("Subscription", style="magenta")
+    table.add_column("Worker's count",  justify="right", style="green")
+
+    for sub in router_stats.subscriptions:
+        table.add_row("key", f"{sub.key=}", f"{len(sub.nodes)=}")
+        table.add_row("hash", f"{sub.hash=}")
+        table.add_row("hits", f"{sub.hits=}")
+        table.add_row("sni_enabled", f"{sub.sni_enabled=}")
+        #print(f"{sub.key=}")
+        for n in sub.nodes:
+            table.add_row("node's name", f"{n.name=}")
+            # table.add_row("node2", f"{n.name=}")
+
+
+    console = Console()
+    console.print(table)
+
+
+
+
+    # for field, value in router_stats.__dict__.items():
+    #     table.add_row(field, str(value))
+    #
+    #
+    #
+    # console = Console()
+    # console.print(table)
+    #
+    # for i in router_stats.__dict__.items():
+    #     print(i)
+
+

requirements.txt

@@ -0,0 +1,4 @@
+ipdb
+uwsgi
+rich
+pydantic

stats.py

@@ -0,0 +1,207 @@
+from pathlib import Path
+
+import pydantic
+
+
+class VirtualHost(pydantic.BaseModel):
+    address: str
+    certificate_path: Path
+    certificate_key: Path
+    server_names: list[str]
+    protocol: str = "https"
+    static_files_mapping: dict = {}
+
+    @property
+    def is_https(self):
+        return all([
+            self.certificate_key,
+            self.certificate_path
+        ])
+
+
+class Router(pydantic.BaseModel):
+    router_id: str
+    subscription_server_address: str
+    app_name: str
+
+    @pydantic.computed_field
+    def subscription_server_port(self) -> int:
+        try:
+            return int(self.subscription_server_address.split(":")[-1])
+        except IndexError:
+            return 0
+
+    @pydantic.computed_field
+    def subscription_server_key(self) -> str:
+        # return f"{self.app_name}.pikesquares.dev:{self.subscription_server_port}"
+        print("subscription_server_key")
+        return f"{self.app_name}.pikesquares.dev"
+
+    @pydantic.computed_field
+    def subscription_server_protocol(self) -> str:
+        return "http" if str(self.subscription_server_port).startswith("9") else "https"
+
+
+class WsgiAppOptions(pydantic.BaseModel):
+    root_dir: Path
+    pyvenv_dir: Path
+    wsgi_file: Path
+    wsgi_module: str
+    routers: list[Router] = []
+    project_id: str
+    workers: int = 3
+
+
+class RouterNode(pydantic.BaseModel):
+    model_config = pydantic.ConfigDict(strict=True)
+    name: str
+    modifier1: int
+    modifier2: int
+    last_check: int
+    pid: int
+    uid: int
+    gid: int
+    requests: int
+    last_requests: int
+    tx: int
+    rx: int
+    cores: int
+    load: int
+    weight: int
+    wrr: int
+    ref: int
+    failcnt: int
+    death_mark: int
+
+
+class RouterSubscription(pydantic.BaseModel):
+    model_config = pydantic.ConfigDict(strict=True)
+    key: str  # 'muffled-castle.pikesquares.dev:5700'
+    hash: int
+    hits: int = pydantic.Field(ge=0)
+    sni_enabled: int
+    nodes: list[RouterNode]
+
+
+class RouterStats(pydantic.BaseModel):
+    model_config = pydantic.ConfigDict(strict=True)
+    version: str
+    pid: int = pydantic.Field(ge=0)
+    uid: int = pydantic.Field(ge=0)
+    gid: int = pydantic.Field(ge=0)
+    cwd: str
+    active_sessions: int = pydantic.Field(ge=0)
+    http: list[str]  # ['0.0.0.0:8034', '127.0.0.1:5700'],
+    subscriptions: list[RouterSubscription]
+    cheap: int = pydantic.Field(ge=0)
+
+
+class DeviceAppStats(pydantic.BaseModel):
+    model_config = pydantic.ConfigDict(strict=True)
+    id: str  # "project_sandbox.json",
+    pid: int
+    born: int
+    last_mod: int
+    last_heartbeat: int
+    loyal: int
+    ready: int
+    accepting: int
+    last_loyal: int
+    last_ready: int
+    last_accepting: int
+    first_run: int
+    last_run: int
+    cursed: int
+    zerg: int
+    on_demand: str
+    uid: int
+    gid: int
+    monitor: str
+    respawns: int
+
+
+class DeviceStats(pydantic.BaseModel):
+    model_config = pydantic.ConfigDict(strict=True)
+    version: str
+    pid: int = pydantic.Field(ge=0)
+    uid: int = pydantic.Field(ge=0)
+    gid: int = pydantic.Field(ge=0)
+    cwd: str
+    emperor: list[str]
+    emperor_tyrant: int
+    throttle_level: int
+    vassals: list[DeviceAppStats]
+    blacklist: list
+
+
+class SocketStats(pydantic.BaseModel):
+    model_config = pydantic.ConfigDict(strict=True)
+    name: str  # "127.0.0.1:4017"
+    proto: str  # "uwsgi"
+    queue: int
+    max_queue: int
+    shared: int
+    can_offload: int
+
+
+class WorkerAppStats(pydantic.BaseModel):
+    model_config = pydantic.ConfigDict(strict=True)
+    id: int
+    modifier1: int
+    mountpoint: str
+    startup_time: int
+    requests: int
+    exceptions: int
+    chdir: str
+
+
+class WorkerCoresStats(pydantic.BaseModel):
+    model_config = pydantic.ConfigDict(strict=True)
+    id: int
+    requests: int
+    static_requests: int
+    routed_requests: int
+    offloaded_requests: int
+    write_errors: int
+    read_errors: int
+    in_request: int
+    vars: list
+    req_info: dict
+
+
+class WorkerStats(pydantic.BaseModel):
+    model_config = pydantic.ConfigDict(strict=True)
+    id: int
+    pid: int
+    accepting: int
+    requests: int
+    delta_requests: int
+    exceptions: int
+    harakiri_count: int
+    signals: int
+    signal_queue: int
+    status: str  # "idle",
+    rss: int
+    vsz: int
+    running_time: int
+    last_spawn: int
+    respawn_count: int
+    tx: int
+    avg_rt: int
+    apps: list[WorkerAppStats]
+
+
+class AppStats(pydantic.BaseModel):
+    model_config = pydantic.ConfigDict(strict=True)
+    version: str
+    listen_queue: int
+    listen_queue_errors: int
+    signal_queue: int
+    load: int
+    pid: int
+    uid: int
+    gid: int
+    cwd: str
+    locks: list[dict[str, int]]
+    sockets: list[SocketStats]
+    workers: list[WorkerStats]

sub1.ini

@@ -0,0 +1,8 @@
+[uwsgi]
+strict=true
+master=true
+touch-reload=sub1.ini
+vacuum=true
+http=0.0.0.0:8934
+http-subscription-server=127.0.0.1:9700
+http-stats=run/sub1-stats.sock

sub2.ini

@@ -0,0 +1,8 @@
+[uwsgi]
+strict=true
+master=true
+touch-reload=sub2.ini
+vacuum=true
+http=0.0.0.0:8935
+http-subscription-server=127.0.0.1:9701
+http-stats=run/sub2-stats.sock

text.txt

@@ -0,0 +1,4 @@
+My name is Julia.
+I live in NYC.
+I study Python.
+In my spare time I go to gym, run and watch movies on projector.