fix: auto-save state on Ctrl+C/SIGTERM, detect interrupted runs
This commit is contained in:
+67
-12
@@ -6,6 +6,8 @@ main.py — Точка входа. Меню выбора режима.
|
|||||||
import sys
|
import sys
|
||||||
import os
|
import os
|
||||||
import argparse
|
import argparse
|
||||||
|
import signal
|
||||||
|
import atexit
|
||||||
|
|
||||||
_PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
_PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||||
if _PROJECT_ROOT not in sys.path:
|
if _PROJECT_ROOT not in sys.path:
|
||||||
@@ -15,6 +17,31 @@ from core.color import banner, menu, prompt, info, error as cerror, warn
|
|||||||
from core import state
|
from core import state
|
||||||
|
|
||||||
|
|
||||||
|
def _save_on_exit():
|
||||||
|
"""Сохраняет состояние при любом завершении (Ctrl+C, SIGTERM, os._exit)."""
|
||||||
|
state.flush_state()
|
||||||
|
|
||||||
|
|
||||||
|
def _handle_sigterm(signum, frame):
|
||||||
|
"""Обработчик SIGTERM (kill, systemd stop)."""
|
||||||
|
state.set_interrupted()
|
||||||
|
state.flush_state()
|
||||||
|
sys.exit(128 + signum)
|
||||||
|
|
||||||
|
|
||||||
|
def _handle_keyboard_interrupt():
|
||||||
|
"""Обработчик Ctrl+C — сохраняет состояние до выхода."""
|
||||||
|
state.set_interrupted()
|
||||||
|
state.flush_state()
|
||||||
|
print()
|
||||||
|
warn("Прервано (Ctrl+C). Состояние сохранено. Запустите docker-migrate --resume для продолжения.")
|
||||||
|
sys.exit(130)
|
||||||
|
|
||||||
|
# Регистрируем сохранение state при любом завершении
|
||||||
|
atexit.register(_save_on_exit)
|
||||||
|
signal.signal(signal.SIGTERM, _handle_sigterm)
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
parser = argparse.ArgumentParser(description="Docker Service Migration Tool")
|
parser = argparse.ArgumentParser(description="Docker Service Migration Tool")
|
||||||
parser.add_argument("--mode", choices=["source", "target"], help="Режим работы")
|
parser.add_argument("--mode", choices=["source", "target"], help="Режим работы")
|
||||||
@@ -42,44 +69,66 @@ def main():
|
|||||||
|
|
||||||
if args.mode == "source":
|
if args.mode == "source":
|
||||||
from source.source import run_source_mode
|
from source.source import run_source_mode
|
||||||
|
try:
|
||||||
run_source_mode()
|
run_source_mode()
|
||||||
|
except KeyboardInterrupt:
|
||||||
|
_handle_keyboard_interrupt()
|
||||||
sys.exit(0)
|
sys.exit(0)
|
||||||
elif args.mode == "target":
|
elif args.mode == "target":
|
||||||
if args.remote_dir:
|
if args.remote_dir:
|
||||||
state.set_stage("INIT", mode="target", target_remote_dir=args.remote_dir)
|
state.set_stage("INIT", mode="target", target_remote_dir=args.remote_dir)
|
||||||
from target.target import run_target_mode
|
from target.target import run_target_mode
|
||||||
|
try:
|
||||||
run_target_mode()
|
run_target_mode()
|
||||||
|
except KeyboardInterrupt:
|
||||||
|
_handle_keyboard_interrupt()
|
||||||
sys.exit(0)
|
sys.exit(0)
|
||||||
|
|
||||||
# Интерактивный режим
|
# Интерактивный режим
|
||||||
|
# Проверим, не было ли прерывания в прошлом запуске
|
||||||
|
recovered = state.recover_after_interrupt()
|
||||||
|
if recovered:
|
||||||
|
banner()
|
||||||
|
warn(f"⚠ Обнаружено прерывание прошлого запуска!")
|
||||||
|
info(f" Шаг: {recovered['stage']}")
|
||||||
|
info(f" Режим: {recovered['mode']}")
|
||||||
|
print(f" {recovered['hint']}")
|
||||||
|
print()
|
||||||
|
if confirm("Продолжить с места прерывания", default="y"):
|
||||||
|
try:
|
||||||
|
_do_resume()
|
||||||
|
except KeyboardInterrupt:
|
||||||
|
_handle_keyboard_interrupt()
|
||||||
|
sys.exit(0)
|
||||||
|
else:
|
||||||
|
info("Сбрасываем прерванное состояние. Можете начать заново.")
|
||||||
|
state.set_stage("INIT", mode=None, interrupted_at=None, resumable_hint=None)
|
||||||
|
|
||||||
banner()
|
banner()
|
||||||
while True:
|
while True:
|
||||||
st = state.load_state()
|
st = state.load_state()
|
||||||
has_resume = bool(st.get("mode") and st.get("stage") and st.get("stage") != "INIT")
|
# Это resume — если есть mode, stage != INIT, или было прерывание
|
||||||
|
has_resume = bool(st.get("mode") and st.get("stage") != "INIT")
|
||||||
menu(has_resume=has_resume)
|
menu(has_resume=has_resume)
|
||||||
try:
|
try:
|
||||||
choice = prompt("Ваш выбор", default="0").strip()
|
choice = prompt("Ваш выбор", default="0").strip()
|
||||||
except SystemExit:
|
except EOFError:
|
||||||
break
|
break
|
||||||
|
except KeyboardInterrupt:
|
||||||
|
_handle_keyboard_interrupt()
|
||||||
if choice == "1":
|
if choice == "1":
|
||||||
from source.source import run_source_mode
|
from source.source import run_source_mode
|
||||||
try:
|
try:
|
||||||
run_source_mode()
|
run_source_mode()
|
||||||
except KeyboardInterrupt:
|
except KeyboardInterrupt:
|
||||||
warn("Прервано")
|
_handle_keyboard_interrupt()
|
||||||
except Exception as e:
|
|
||||||
cerror(f"Ошибка: {e}")
|
|
||||||
sys.exit(1)
|
|
||||||
break
|
break
|
||||||
elif choice == "2":
|
elif choice == "2":
|
||||||
from target.target import run_target_mode
|
from target.target import run_target_mode
|
||||||
try:
|
try:
|
||||||
run_target_mode()
|
run_target_mode()
|
||||||
except KeyboardInterrupt:
|
except KeyboardInterrupt:
|
||||||
warn("Прервано")
|
_handle_keyboard_interrupt()
|
||||||
except Exception as e:
|
|
||||||
cerror(f"Ошибка: {e}")
|
|
||||||
sys.exit(1)
|
|
||||||
break
|
break
|
||||||
elif choice == "3":
|
elif choice == "3":
|
||||||
if not has_resume:
|
if not has_resume:
|
||||||
@@ -88,7 +137,7 @@ def main():
|
|||||||
try:
|
try:
|
||||||
_do_resume()
|
_do_resume()
|
||||||
except KeyboardInterrupt:
|
except KeyboardInterrupt:
|
||||||
warn("Прервано")
|
_handle_keyboard_interrupt()
|
||||||
break
|
break
|
||||||
elif choice == "4":
|
elif choice == "4":
|
||||||
_do_status_logs()
|
_do_status_logs()
|
||||||
@@ -107,9 +156,13 @@ def _do_resume():
|
|||||||
if not mode or stage == "INIT":
|
if not mode or stage == "INIT":
|
||||||
info("Нет сохранённого состояния для продолжения")
|
info("Нет сохранённого состояния для продолжения")
|
||||||
return
|
return
|
||||||
|
# Сбрасываем флаг прерывания, если был
|
||||||
|
state.set_stage(stage, interrupted_at=None, resumable_hint=None)
|
||||||
fsm = FSM(mode=mode)
|
fsm = FSM(mode=mode)
|
||||||
try:
|
try:
|
||||||
fsm.resume_from(stage)
|
fsm.resume_from(stage)
|
||||||
|
except KeyboardInterrupt:
|
||||||
|
_handle_keyboard_interrupt()
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
cerror(f"Ошибка при resume: {e}")
|
cerror(f"Ошибка при resume: {e}")
|
||||||
|
|
||||||
@@ -123,8 +176,10 @@ def _do_status_logs():
|
|||||||
print(" 0 Назад")
|
print(" 0 Назад")
|
||||||
try:
|
try:
|
||||||
c = prompt("Выбор", default="0").strip()
|
c = prompt("Выбор", default="0").strip()
|
||||||
except SystemExit:
|
except EOFError:
|
||||||
break
|
break
|
||||||
|
except KeyboardInterrupt:
|
||||||
|
_handle_keyboard_interrupt()
|
||||||
if c == "1":
|
if c == "1":
|
||||||
stmod.show_status()
|
stmod.show_status()
|
||||||
elif c == "2":
|
elif c == "2":
|
||||||
|
|||||||
+44
-5
@@ -7,17 +7,17 @@ state.py — Управление состоянием (FSM) и resume
|
|||||||
import os
|
import os
|
||||||
import json
|
import json
|
||||||
import sys
|
import sys
|
||||||
import time
|
import signal
|
||||||
|
import atexit
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
|
||||||
STATE_FILE = None
|
STATE_FILE = None
|
||||||
|
_LAST_STATE = None
|
||||||
|
|
||||||
def _state_path():
|
def _state_path():
|
||||||
global STATE_FILE
|
global STATE_FILE
|
||||||
if STATE_FILE:
|
if STATE_FILE:
|
||||||
return STATE_FILE
|
return STATE_FILE
|
||||||
# Ищем рядом со скриптом или в /var/lib/migrate
|
|
||||||
candidates = [
|
candidates = [
|
||||||
os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), ".migrate-state.json"),
|
os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), ".migrate-state.json"),
|
||||||
os.path.join(os.path.expanduser("~"), ".migrate", "state.json"),
|
os.path.join(os.path.expanduser("~"), ".migrate", "state.json"),
|
||||||
@@ -40,14 +40,18 @@ def _state_path():
|
|||||||
|
|
||||||
|
|
||||||
def load_state():
|
def load_state():
|
||||||
|
global _LAST_STATE
|
||||||
sp = _state_path()
|
sp = _state_path()
|
||||||
if os.path.isfile(sp):
|
if os.path.isfile(sp):
|
||||||
try:
|
try:
|
||||||
with open(sp, "r", encoding="utf-8") as f:
|
with open(sp, "r", encoding="utf-8") as f:
|
||||||
return json.load(f)
|
data = json.load(f)
|
||||||
|
_LAST_STATE = data
|
||||||
|
return data
|
||||||
except json.JSONDecodeError:
|
except json.JSONDecodeError:
|
||||||
pass
|
pass
|
||||||
return _default_state()
|
_LAST_STATE = _default_state()
|
||||||
|
return _LAST_STATE
|
||||||
|
|
||||||
|
|
||||||
def _default_state():
|
def _default_state():
|
||||||
@@ -61,14 +65,25 @@ def _default_state():
|
|||||||
"target_port": 22,
|
"target_port": 22,
|
||||||
"last_error": None,
|
"last_error": None,
|
||||||
"completed_steps": [],
|
"completed_steps": [],
|
||||||
|
"interrupted_at": None,
|
||||||
"paused": False,
|
"paused": False,
|
||||||
"created_at": datetime.now().isoformat(),
|
"created_at": datetime.now().isoformat(),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def flush_state():
|
||||||
|
"""Сохраняет текущее _LAST_STATE на диск."""
|
||||||
|
global _LAST_STATE
|
||||||
|
sp = _state_path()
|
||||||
|
with open(sp, "w", encoding="utf-8") as f:
|
||||||
|
json.dump(_LAST_STATE, f, indent=2, ensure_ascii=False)
|
||||||
|
|
||||||
|
|
||||||
def save_state(state):
|
def save_state(state):
|
||||||
|
global _LAST_STATE
|
||||||
sp = _state_path()
|
sp = _state_path()
|
||||||
state["updated_at"] = datetime.now().isoformat()
|
state["updated_at"] = datetime.now().isoformat()
|
||||||
|
_LAST_STATE = state
|
||||||
with open(sp, "w", encoding="utf-8") as f:
|
with open(sp, "w", encoding="utf-8") as f:
|
||||||
json.dump(state, f, indent=2, ensure_ascii=False)
|
json.dump(state, f, indent=2, ensure_ascii=False)
|
||||||
|
|
||||||
@@ -112,6 +127,26 @@ def clear_error():
|
|||||||
save_state(state)
|
save_state(state)
|
||||||
|
|
||||||
|
|
||||||
|
def set_interrupted():
|
||||||
|
"""Помечает, что скрипт был прерван (Ctrl+C, SIGTERM)."""
|
||||||
|
state = load_state()
|
||||||
|
state["interrupted_at"] = datetime.now().isoformat()
|
||||||
|
state["resumable_hint"] = f"Скрипт был прерван на шаге '{state.get('stage', 'INIT')}'. Запустите: docker-migrate --resume"
|
||||||
|
save_state(state)
|
||||||
|
|
||||||
|
|
||||||
|
def recover_after_interrupt():
|
||||||
|
"""Проверяет, был ли предыдущий запуск прерван. Если да — сбрасывает флаг."""
|
||||||
|
state = load_state()
|
||||||
|
if state.get("interrupted_at"):
|
||||||
|
return {
|
||||||
|
"stage": state.get("stage"),
|
||||||
|
"mode": state.get("mode"),
|
||||||
|
"hint": state.pop("resumable_hint", ""),
|
||||||
|
}
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
def show_status():
|
def show_status():
|
||||||
from core.color import info, warn, error as cerror, bold, divider
|
from core.color import info, warn, error as cerror, bold, divider
|
||||||
state = load_state()
|
state = load_state()
|
||||||
@@ -123,6 +158,9 @@ def show_status():
|
|||||||
info(f"Completed steps: {', '.join(state.get('completed_steps', [])) or '(none)'}")
|
info(f"Completed steps: {', '.join(state.get('completed_steps', [])) or '(none)'}")
|
||||||
if state.get("paused"):
|
if state.get("paused"):
|
||||||
warn("Скрипт на паузе из-за ошибки")
|
warn("Скрипт на паузе из-за ошибки")
|
||||||
|
if state.get("interrupted_at"):
|
||||||
|
warn(f"Последний запуск был прерван в {state['interrupted_at']} на шаге '{state.get('stage', 'INIT')}'")
|
||||||
|
print(f" Запустите: docker-migrate --resume")
|
||||||
if state.get("manifest_path"):
|
if state.get("manifest_path"):
|
||||||
info(f"Manifest: {state['manifest_path']}")
|
info(f"Manifest: {state['manifest_path']}")
|
||||||
if state.get("archive_path"):
|
if state.get("archive_path"):
|
||||||
@@ -154,3 +192,4 @@ def show_logs():
|
|||||||
if err.get("suggestion"):
|
if err.get("suggestion"):
|
||||||
print(f"\nРекомендация:")
|
print(f"\nРекомендация:")
|
||||||
print(err["suggestion"])
|
print(err["suggestion"])
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user