diff --git a/MP2/.gitignore b/MP2/.gitignore index 528dc4e..e69de29 100644 --- a/MP2/.gitignore +++ b/MP2/.gitignore @@ -1 +0,0 @@ -scripts \ No newline at end of file diff --git a/MP2/peer.py b/MP2/peer.py index ea24cfb..3a1caa9 100644 --- a/MP2/peer.py +++ b/MP2/peer.py @@ -31,13 +31,13 @@ GLOBAL_PORT = 8989 MY_IP = socket.gethostbyname(socket.gethostname()) HEARTBEAT_INTERVAL = 0.25 -GOSSIP_INTERVAL = 0.35 +GOSSIP_INTERVAL = 0.25 JOIN_RETRY_INTERVAL = 1.0 GOSSIP_FANOUT = 2 -GOSSIP_FAILURE_TIMEOUT = 2.8 -GOSSIP_S_SUSPECT_TIMEOUT = 1.2 -GOSSIP_S_TIMEOUT_SUSPECT = 1.6 +GOSSIP_FAILURE_TIMEOUT = 3 +GOSSIP_S_SUSPECT_TIMEOUT = 1.5 +GOSSIP_S_TIMEOUT_SUSPECT = 1.5 GOSSIP_FAILURE_TIMEOUT_CLEANUP = 10.0 diff --git a/MP2/scripts/cmd.sh b/MP2/scripts/cmd.sh new file mode 100755 index 0000000..f7d2c28 --- /dev/null +++ b/MP2/scripts/cmd.sh @@ -0,0 +1,405 @@ +#!/usr/bin/env bash + +SESSION="cs425-debug" +CMDS_WINDOW="cmds" + +NUM_SERVERS=10 + +PORT=8989 + +WORKDIR="fa26_cs425_.group-mp_.team-01/MP2" +SCRIPT="peer.py" + +SSH_OPTS="-o BatchMode=yes \ +-o ConnectTimeout=2 \ +-o ServerAliveInterval=1 \ +-o ServerAliveCountMax=1" + + +# HELPERS + + +expand_targets() { + + if [ "$1" = "all" ]; then + seq 1 "$NUM_SERVERS" + else + echo "$@" + fi +} + + +get_pane() { + + local vm="$1" + + # VM1 -> pane 0 + # VM2 -> pane 1 + + echo "$SESSION:$CMDS_WINDOW.$((vm - 1))" +} + + +is_running() { + + local vm="$1" + + ssh $SSH_OPTS "server$vm" \ + "pgrep -f '[p]ython3.*$SCRIPT' >/dev/null" \ + 2>/dev/null +} + + + +# START + + +start_vm() { + + local vm="$1" + local pane + + pane=$(get_pane "$vm") + + if ! tmux has-session \ + -t "$SESSION" \ + 2>/dev/null + then + echo "VM$vm: tmux session not running" + return 1 + fi + + echo "starting VM$vm..." + + # Kill old process on remote VM + + ssh $SSH_OPTS "server$vm" \ + "fuser -k -9 $PORT/udp 2>/dev/null || true; \ + fuser -k -9 $PORT/tcp 2>/dev/null || true; \ + pkill -9 -f '[p]ython3.*$SCRIPT' 2>/dev/null || true" \ + >/dev/null 2>&1 + + sleep 0.3 + + # Reset pane + + tmux send-keys \ + -t "$pane" \ + C-c + + sleep 0.05 + + tmux send-keys \ + -t "$pane" \ + "clear" \ + C-m + + # Start peer.py + + tmux send-keys \ + -t "$pane" \ + "ssh -t $SSH_OPTS server$vm 'cd \$HOME/$WORKDIR && MACHINE_INDEX=$vm python3 $SCRIPT'" \ + C-m +} + + + +# KILL + + +kill_vm() { + + local vm="$1" + + echo "killing VM$vm..." + + ssh $SSH_OPTS "server$vm" \ + "fuser -k -9 $PORT/udp 2>/dev/null || true; \ + fuser -k -9 $PORT/tcp 2>/dev/null || true; \ + pkill -9 -f '[p]ython3.*$SCRIPT' 2>/dev/null || true" \ + >/dev/null 2>&1 +} + + + +# COMMAND + + +cmd_to_vm() { + + local vm="$1" + shift + + local command="$*" + local pane + local pane_command + + pane=$(get_pane "$vm") + + if ! is_running "$vm"; then + echo "VM$vm skipped (STOPPED)" + return + fi + + pane_command=$( + tmux display-message \ + -p \ + -t "$pane" \ + '#{pane_current_command}' \ + 2>/dev/null + ) + + if [ "$pane_command" != "ssh" ]; then + echo "VM$vm skipped (pane not attached)" + return + fi + + echo "VM$vm <- $command" + + tmux send-keys \ + -t "$pane" \ + "$command" \ + C-m +} + + + +# LIST + + +list_vm() { + + local vm="$1" + + echo + echo "========== VM$vm ==========" + + ssh $SSH_OPTS "server$vm" \ + "cat \$HOME/$WORKDIR/membership_list.txt" \ + 2>/dev/null +} + + + +# LOG + + +log_vm() { + + local vm="$1" + + echo + echo "========== VM$vm ==========" + + ssh $SSH_OPTS "server$vm" \ + "tail -n 30 \$HOME/$WORKDIR/machine.$vm.log" \ + 2>/dev/null +} + + + +# STATUS + + +status_vm() { + + local vm="$1" + local pid + + pid=$( + ssh $SSH_OPTS "server$vm" \ + "pgrep -f '[p]ython3.*$SCRIPT' | tr '\n' ' '" \ + 2>/dev/null + ) + + if [ -n "$pid" ]; then + echo "VM$vm RUNNING, PID(s): $pid" + else + echo "VM$vm STOPPED" + fi +} + + + +# MAIN + + +action="$1" +shift || true + +case "$action" in + + +# START + + +start) + + if [ "$#" -eq 0 ]; then + echo "Usage: ./cmd.sh start " + exit 1 + fi + + targets=$(expand_targets "$@") + + for vm in $targets; do + start_vm "$vm" & + done + + wait + + sleep 0.5 + + for vm in $targets; do + status_vm "$vm" & + done + + wait + ;; + + + +# KILL + + +kill) + + if [ "$#" -eq 0 ]; then + echo "Usage: ./cmd.sh kill " + exit 1 + fi + + targets=$(expand_targets "$@") + + for vm in $targets; do + kill_vm "$vm" & + done + + wait + ;; + + + +# CMD + + +cmd) + + target="$1" + shift || true + + if [ -z "$target" ] || [ "$#" -eq 0 ]; then + echo "Usage:" + echo "./cmd.sh cmd \"command\"" + exit 1 + fi + + if [ "$target" = "all" ]; then + + for vm in $(seq 1 "$NUM_SERVERS"); do + cmd_to_vm "$vm" "$@" & + done + + wait + + else + + cmd_to_vm "$target" "$@" + + fi + ;; + + + +# LIST + + +list) + + if [ "$#" -eq 0 ]; then + targets=$(seq 1 "$NUM_SERVERS") + else + targets=$(expand_targets "$@") + fi + + for vm in $targets; do + list_vm "$vm" & + done + + wait + ;; + + + +# LOG + + +log) + + if [ "$#" -eq 0 ]; then + targets=$(seq 1 "$NUM_SERVERS") + else + targets=$(expand_targets "$@") + fi + + for vm in $targets; do + log_vm "$vm" & + done + + wait + ;; + + + +# STATUS + + +status) + + if [ "$#" -eq 0 ]; then + targets=$(seq 1 "$NUM_SERVERS") + else + targets=$(expand_targets "$@") + fi + + for vm in $targets; do + status_vm "$vm" & + done + + wait + ;; + + + +# HELP + + +*) + + echo + echo "Usage:" + echo + echo " ./cmd.sh start " + echo " ./cmd.sh kill " + echo " ./cmd.sh cmd \"command\"" + echo " ./cmd.sh list [all|N...]" + echo " ./cmd.sh log [all|N...]" + echo " ./cmd.sh status [all|N...]" + echo + echo "Examples:" + echo + echo " ./cmd.sh start 1" + echo " ./cmd.sh start 2 3 4 5" + echo " ./cmd.sh start 6 7 8 9 10" + echo + echo " ./cmd.sh cmd all \"switch suspect\"" + echo " ./cmd.sh cmd all \"switch nosuspect\"" + echo " ./cmd.sh cmd all \"set_drop_rate 10\"" + echo " ./cmd.sh cmd all \"display_protocol\"" + echo + echo " ./cmd.sh kill 8 9" + echo " ./cmd.sh log 8 9" + echo + exit 1 + ;; +esac \ No newline at end of file diff --git a/MP2/scripts/pretty_membership.py b/MP2/scripts/pretty_membership.py new file mode 100644 index 0000000..5e4bc77 --- /dev/null +++ b/MP2/scripts/pretty_membership.py @@ -0,0 +1,254 @@ +#!/usr/bin/env python3 + +import sys +import ast +import time + + +IP_TO_VM = { + "172.22.156.3": "1", + "172.22.158.2": "2", + "172.22.94.2": "3", + "172.22.156.4": "4", + "172.22.158.3": "5", + "172.22.94.3": "6", + "172.22.156.5": "7", + "172.22.158.4": "8", + "172.22.94.4": "9", + "172.22.156.6": "10", +} + + + +# COLORS + + +GREEN = "\033[92m" +YELLOW = "\033[93m" +RED = "\033[91m" +CYAN = "\033[96m" +WHITE = "\033[97m" +GRAY = "\033[90m" + +BOLD = "\033[1m" +RESET = "\033[0m" + +CLEAR_LINE = "\033[K" + + +STATE_STYLE = { + "ALIVE": (GREEN, "A"), + "SUSPECT": (YELLOW, "S"), + "FAILED": (RED, "F"), +} + + +def render(lines): + + + # STOPPED / SSH ERROR + + + for line in lines: + if line.startswith("**STOPPED**"): + + reason = line.replace( + "**STOPPED**", + "" + ).strip() + + return ( + f"{CLEAR_LINE}" + f"{BOLD}{RED}>>> STOPPED <<<{RESET}" + f"{CLEAR_LINE}\n" + f"{GRAY}{reason}{RESET}" + f"{CLEAR_LINE}\n" + ) + + + # PARSE + + + members = [] + + mode = "?" + drop_rate = "?" + + now = time.time() + + for raw_line in lines: + + line = raw_line.strip() + + if not line: + continue + + # ---------------------------------------------------- + # MODE / DROP RATE LINE + # ---------------------------------------------------- + + if line.startswith("Mode:"): + + try: + pieces = line.split(",") + + mode = pieces[0].split(":", 1)[1].strip() + drop_rate = pieces[1].split(":", 1)[1].strip() + + except Exception: + pass + + continue + + # ---------------------------------------------------- + # MEMBERSHIP ENTRY + # ---------------------------------------------------- + + try: + + id_text, data_text = line.split(": ", 1) + + member_id = ast.literal_eval(id_text) + member_data = ast.literal_eval(data_text) + + ip, port, timestamp = member_id + + local_last_seen = member_data.get( + "local_last_seen", + now + ) + + members.append({ + "vm": IP_TO_VM.get(ip, "?"), + + "hb": member_data.get( + "heart_beat", + 0 + ), + + "state": member_data.get( + "state", + "?" + ).upper(), + + "age": max( + 0.0, + now - local_last_seen + ), + + "inc": member_data.get( + "incarnation_counter", + 0 + ), + }) + + except Exception: + continue + + + # HEADER + + + out = [] + + mode_upper = str(mode).upper() + + if mode_upper == "SUSPECT": + mode_color = YELLOW + elif mode_upper == "NOSUSPECT": + mode_color = GREEN + else: + mode_color = GRAY + + out.append( + f"{BOLD}{CYAN}MODE:{RESET} " + f"{mode_color}{BOLD}{mode_upper}{RESET} " + f"{GRAY}|{RESET} " + f"{BOLD}{CYAN}DROP:{RESET} " + f"{WHITE}{drop_rate}%{RESET}" + f"{CLEAR_LINE}" + ) + + + # NO DATA + + + if not members: + + out.append( + f"{BOLD}{YELLOW}" + f"waiting for membership..." + f"{RESET}" + f"{CLEAR_LINE}" + ) + + return "\n".join(out) + "\n" + + + # SORT + + + def vm_number(member): + + try: + return int(member["vm"]) + + except Exception: + return 999 + + members.sort( + key=vm_number + ) + + + # TABLE HEADER + + + out.append( + f"{BOLD}{CYAN}" + f"{'VM':<3} " + f"{'S':<2} " + f"{'HB':<6} " + f"{'AGE':<5} " + f"{'INC'}" + f"{RESET}" + f"{CLEAR_LINE}" + ) + + + # MEMBERS + + + for member in members: + + color, code = STATE_STYLE.get( + member["state"], + (GRAY, "?") + ) + + out.append( + f"{WHITE}" + f"{member['vm']:<3}" + f"{RESET} " + + f"{color}{BOLD}" + f"{code:<2}" + f"{RESET} " + + f"{member['hb']:<6} " + + f"{member['age']:>4.1f} " + + f"{member['inc']}" + + f"{CLEAR_LINE}" + ) + + return "\n".join(out) + "\n" + + +print( + render(list(sys.stdin)), + end="", + flush=True +) \ No newline at end of file diff --git a/MP2/scripts/view.sh b/MP2/scripts/view.sh new file mode 100755 index 0000000..266f099 --- /dev/null +++ b/MP2/scripts/view.sh @@ -0,0 +1,147 @@ +#!/usr/bin/env bash + +SESSION="cs425-debug" +NUM_SERVERS=10 + +WORKDIR="fa26_cs425_.group-mp_.team-01/MP2" +FORMATTER="$(cd "$(dirname "$0")" && pwd)/pretty_membership.py" + +SSH_OPTS="-o ConnectTimeout=2 \ +-o ServerAliveInterval=1 \ +-o ServerAliveCountMax=1 \ +-o BatchMode=yes" + + +# RESET TMUX + + +tmux kill-session -t "$SESSION" 2>/dev/null || true + +tmux new-session -d -s "$SESSION" -n "display" + +tmux set-option -t "$SESSION" pane-border-status top +tmux set-option -t "$SESSION" pane-border-format " #{pane_title} " + + +# DISPLAY COMMAND + + +make_display_cmd() { + local vm="$1" + + cat </dev/null || echo 0); + AGE=\$((NOW - MTIME)); + + if [ "\$AGE" -gt 3 ]; then + echo "**STOPPED** \${AGE}s since last update"; + else + cat "\$FILE"; + fi; + fi' 2>&1 \ + || echo "**STOPPED** SSH unavailable" + } | python3 "$FORMATTER" + + printf '\033[J' + sleep 0.75 +done +EOF +} + + +# DISPLAY WINDOW + + +tmux select-pane \ + -t "$SESSION:display.0" \ + -T "VM1" + +tmux send-keys \ + -t "$SESSION:display.0" \ + "$(make_display_cmd 1)" \ + C-m + +for i in $(seq 2 "$NUM_SERVERS"); do + + PANE=$( + tmux split-window \ + -t "$SESSION:display" \ + -P \ + -F '#{pane_id}' + ) + + tmux select-layout \ + -t "$SESSION:display" \ + tiled + + tmux select-pane \ + -t "$PANE" \ + -T "VM$i" + + tmux send-keys \ + -t "$PANE" \ + "$(make_display_cmd "$i")" \ + C-m +done + +tmux select-layout \ + -t "$SESSION:display" \ + tiled + + +# COMMAND WINDOW + + +tmux new-window \ + -t "$SESSION" \ + -n "cmds" + +tmux set-window-option \ + -t "$SESSION:cmds" \ + pane-base-index 0 + +tmux select-pane \ + -t "$SESSION:cmds.0" \ + -T "VM1" + +for i in $(seq 2 "$NUM_SERVERS"); do + + PANE=$( + tmux split-window \ + -t "$SESSION:cmds" \ + -P \ + -F '#{pane_id}' + ) + + tmux select-layout \ + -t "$SESSION:cmds" \ + tiled + + tmux select-pane \ + -t "$PANE" \ + -T "VM$i" +done + +tmux select-layout \ + -t "$SESSION:cmds" \ + tiled + + +# OPEN DISPLAY + + +tmux select-window \ + -t "$SESSION:display" + +tmux attach-session \ + -t "$SESSION" \ No newline at end of file