208 lines
8.1 KiB
Python
208 lines
8.1 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
从可见浏览器中提取token，然后用API方式抓取所有数据
|
|
"""
|
|
import asyncio, json, os, time
|
|
from playwright.async_api import async_playwright
|
|
|
|
BASE = "http://39.101.149.173"
|
|
USERNAME = "admin"
|
|
PASSWORD = "ndui#Emdfj23123"
|
|
TOKEN_FILE = "/tmp/wms_token.json"
|
|
OUTPUT_DIR = os.path.expanduser("~/Documents/柏悦")
|
|
|
|
async def main():
|
|
async with async_playwright() as p:
|
|
browser = await p.chromium.launch(headless=False, args=["--start-maximized"])
|
|
context = await browser.new_context(viewport={"width": 1440, "height": 900})
|
|
page = await context.new_page()
|
|
|
|
print("=== 打开登录页 ===")
|
|
await page.goto(f"{BASE}/user/login?redirect=%2Fwms%2FByInstoreList", timeout=30000)
|
|
await page.wait_for_timeout(2000)
|
|
|
|
# 填充
|
|
inputs = page.locator("input[type='text']")
|
|
pwd = page.locator("input[type='password']")
|
|
await inputs.nth(0).fill(USERNAME)
|
|
await pwd.fill(PASSWORD)
|
|
|
|
print("👉 请在浏览器中输入验证码并点击确定...")
|
|
|
|
try:
|
|
await page.wait_for_function(
|
|
"() => !window.location.pathname.includes('login')",
|
|
timeout=300000
|
|
)
|
|
print("✅ 登录成功!")
|
|
except:
|
|
print("❌ 超时")
|
|
await browser.close()
|
|
return
|
|
|
|
await page.wait_for_timeout(3000)
|
|
|
|
# ========== 抓取完整菜单结构 ==========
|
|
print("\n=== 抓取菜单 ===")
|
|
menu_structure = {}
|
|
page_list = [] # [(display_name, url_path)]
|
|
|
|
top_menus = page.locator(".ant-menu-submenu-title")
|
|
for i in range(await top_menus.count()):
|
|
try:
|
|
name = (await top_menus.nth(i).inner_text()).strip().split("\n")[0].strip()
|
|
if not name or len(name) > 20:
|
|
continue
|
|
|
|
await top_menus.nth(i).click()
|
|
await page.wait_for_timeout(500)
|
|
|
|
subs = []
|
|
sub_items = page.locator(".ant-menu-submenu-open .ant-menu-item")
|
|
for j in range(await sub_items.count()):
|
|
sn = (await sub_items.nth(j).inner_text()).strip().split("\n")[0]
|
|
if sn and len(sn) < 30:
|
|
subs.append(sn)
|
|
|
|
menu_structure[name] = subs
|
|
print(f" {name}: {subs}")
|
|
|
|
await top_menus.nth(i).click()
|
|
await page.wait_for_timeout(300)
|
|
except:
|
|
pass
|
|
|
|
# ========== 逐页截图抓取 ==========
|
|
print(f"\n=== 逐页抓取 ===")
|
|
|
|
# 构建所有要访问的页面
|
|
all_pages = [("首页", "/wms/Dashboard")]
|
|
|
|
for menu_name, subs in menu_structure.items():
|
|
for sub in subs:
|
|
# 推断URL路径
|
|
path_map = {
|
|
"入库单列表": "/wms/ByInstoreList",
|
|
"入库单测量": "/wms/ByInstoreList4Measured",
|
|
"测量数据审核": "/wms/ByInstoreList4MeasuredAudit",
|
|
"接收审核": "/wms/ByInstoreList4ReceiveAudit",
|
|
"客户管理": "/wms/ByCustomerList",
|
|
"出库单列表": "/wms/ByOutstoreList",
|
|
"派送单列表": "/wms/ByDispatchList",
|
|
"BOL管理": "/wms/ByBolList",
|
|
"货柜管理": "/wms/ByContainerList",
|
|
"库存管理": "/wms/ByInventoryList",
|
|
"财务结算": "/wms/ByFinanceList",
|
|
"托盘管理": "/wms/ByTrayList",
|
|
"用户管理": "/system/UserList",
|
|
"角色管理": "/system/RoleList",
|
|
"菜单管理": "/system/MenuList",
|
|
"字典管理": "/system/DictList",
|
|
"参数配置": "/system/ConfigList",
|
|
"通用配置": "/system/ConfigList",
|
|
"报表设计": "/wms/Dashboard", # 待确认
|
|
"消息中心": "/wms/Dashboard",
|
|
"常见案例": "/wms/Dashboard",
|
|
"在线开发": "/wms/Dashboard",
|
|
"系统监控": "/wms/Dashboard",
|
|
"统计报表": "/wms/Dashboard",
|
|
}
|
|
path = path_map.get(sub, "")
|
|
if path:
|
|
all_pages.append((f"{menu_name} > {sub}", path))
|
|
|
|
# 去重
|
|
seen = set()
|
|
unique_pages = []
|
|
for name, path in all_pages:
|
|
if path not in seen:
|
|
seen.add(path)
|
|
unique_pages.append((name, path))
|
|
|
|
pages_data = {}
|
|
success_count = 0
|
|
|
|
for idx, (name, path) in enumerate(unique_pages, 1):
|
|
try:
|
|
url = BASE + path
|
|
print(f"[{idx}/{len(unique_pages)}] {name}", end=" ... ")
|
|
|
|
await page.goto(url, timeout=15000)
|
|
await page.wait_for_timeout(2500)
|
|
|
|
if "login" in page.url.lower():
|
|
print("会话过期!")
|
|
break
|
|
|
|
# 抓表格列头
|
|
headers = []
|
|
th_selectors = [".ant-table-thead th", "thead th", "table th"]
|
|
for ths in th_selectors:
|
|
els = page.locator(ths)
|
|
for i in range(min(await els.count(), 30)):
|
|
try:
|
|
h = (await els.nth(i).inner_text()).strip()
|
|
if h and h not in headers:
|
|
headers.append(h)
|
|
except:
|
|
pass
|
|
|
|
# 搜索字段
|
|
fields = []
|
|
labels = page.locator("label, .ant-form-item-label")
|
|
for i in range(min(await labels.count(), 25)):
|
|
try:
|
|
t = (await labels.nth(i).inner_text()).strip()
|
|
if t and t not in fields and len(t) < 30:
|
|
fields.append(t)
|
|
except:
|
|
pass
|
|
|
|
# 操作按钮
|
|
buttons = []
|
|
btns = page.locator("button:not([class*='pagination']):not([class*='quick-jumper'])")
|
|
for i in range(min(await btns.count(), 20)):
|
|
try:
|
|
b = (await btns.nth(i).inner_text()).strip()
|
|
if b and b not in buttons and len(b) < 25:
|
|
buttons.append(b)
|
|
except:
|
|
pass
|
|
|
|
pages_data[name] = {
|
|
"url": url,
|
|
"title": await page.title(),
|
|
"table_headers": headers,
|
|
"search_fields": fields,
|
|
"buttons": buttons,
|
|
}
|
|
|
|
print(f"✅ headers:{len(headers)} fields:{len(fields)} btns:{len(buttons)}")
|
|
success_count += 1
|
|
|
|
except Exception as e:
|
|
print(f"❌ {str(e)[:80]}")
|
|
pages_data[name] = {"error": str(e)[:200]}
|
|
|
|
# 保存数据
|
|
output = {
|
|
"system_name": "柏悦物流管理系统 (Baiyue WMS)",
|
|
"scrape_time": time.strftime("%Y-%m-%d %H:%M:%S"),
|
|
"menu_structure": menu_structure,
|
|
"pages": pages_data,
|
|
"pages_count": success_count,
|
|
}
|
|
|
|
out_path = os.path.join(OUTPUT_DIR, "wms_full_data.json")
|
|
with open(out_path, "w", encoding="utf-8") as f:
|
|
json.dump(output, f, indent=2, ensure_ascii=False)
|
|
|
|
print(f"\n✅ 完成! {success_count}/{len(unique_pages)} 页面成功抓取")
|
|
print(f"输出: {out_path}")
|
|
|
|
print("\n浏览器10秒后关闭...")
|
|
await asyncio.sleep(10)
|
|
await browser.close()
|
|
|
|
asyncio.run(main())
|