import subprocess
import urllib.parse
import os
import time
import json
import random
from datetime import datetime


def get_target_file_input():
    """获取目标列表文件路径"""
    while True:
        file_path = input("请输入目标列表文件路径（每行一个目标）：").strip()
        if not file_path:
            print("文件路径不能为空！请重新输入：")
            continue
        if not os.path.exists(file_path):
            print(f"文件不存在：{file_path}！请重新输入：")
            continue
        if not os.path.isfile(file_path):
            print(f"{file_path} 不是有效文件！请重新输入：")
            continue
        with open(file_path, "r", encoding="utf-8") as f:
            targets = [line.strip() for line in f if line.strip()]
        if not targets:
            print(f"文件 {file_path} 中未找到有效目标！请检查文件内容：")
            continue
        print(f"\n成功读取目标列表（共 {len(targets)} 个有效目标，逐个处理）：")
        for i, target in enumerate(targets, 1):
            print(f"  {i}. {target}")
        return file_path, targets


BYPASS_HEADERS = [
    "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language: zh-CN,zh;q=0.9",
    "Connection: keep-alive",
    f"X-Forwarded-For: {random.randint(1, 254)}.{random.randint(1, 254)}.{random.randint(1, 254)}.{random.randint(1, 254)}"
]


def build_curl_command(resource_type, search_keyword, base_url="http://127.0.0.1:16181/query"):
    encoded_keyword = urllib.parse.quote(search_keyword, safe='')
    url = f"{base_url}/{resource_type}?search={encoded_keyword}&pageSize=1000"
    headers = BYPASS_HEADERS

    # 构建curl命令
    curl_cmd = f'curl "{url}" --max-time 60'
    for header in headers:
        curl_cmd += f' -H "{header}"'

    return curl_cmd


def log_to_file(message, resource_type, target=""):
    """日志记录"""
    log_dir = "request_logs"
    os.makedirs(log_dir, exist_ok=True)
    log_file = f"{log_dir}/single_requests_{datetime.now().strftime('%Y%m%d')}.log"
    target_prefix = f"[目标：{target}] " if target else ""
    with open(log_file, "a", encoding="utf-8") as f:
        f.write(f"[{datetime.now().strftime('%H:%M:%S')}] {target_prefix}{message}\n")


def extract_field(response_content, resource_type, target):
    if not response_content.strip():
        log_to_file(f"响应内容为空", resource_type, target)
        return [f"【{target}】{resource_type} 无有效数据"]

    try:
        json_data = json.loads(response_content)
        if json_data.get("code") != 200:
            log_to_file(f"响应失败（code：{json_data.get('code')}", resource_type, target)
            return [f"【{target}】{resource_type} 响应失败（code：{json_data.get('code')}）"]

        list_data = json_data["params"].get("list", [])
        field_map = {"web": "domain", "app": "serviceName", "mapp": "serviceName"}
        target_field = field_map[resource_type]
        fields = [item.get(target_field, "").strip() for item in list_data if item.get(target_field, "").strip()]

        if not fields:
            return [f"【{target}】{resource_type} 未找到有效{target_field}"]
        return fields

    except json.JSONDecodeError:
        log_to_file("JSON解析失败", resource_type, target)
        return [f"【{target}】{resource_type} 数据格式错误"]
    except Exception as e:
        log_to_file(f"提取失败：{str(e)}", resource_type, target)
        return [f"【{target}】{resource_type} 提取异常：{str(e)}"]


def run_single_request(resource_type, target, output_file):
    """单个请求处理（统一延迟为100-120秒）"""
    max_retries = 2
    retry_count = 0

    while retry_count <= max_retries:
        try:
            curl_cmd = build_curl_command(resource_type, target)
            log_to_file(f"执行请求：{curl_cmd}", resource_type, target)
            print(f"\n=== 处理目标【{target}】- 资源类型【{resource_type}】（重试：{retry_count}）===")
            print(f"执行命令：{curl_cmd}")

            result = subprocess.run(
                curl_cmd,
                shell=True,
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE,
                text=True,
                timeout=30
            )

            # WAF拦截检测
            if "安全验证" in result.stdout or "验证码" in result.stdout:
                log_to_file("被WAF拦截（检测到验证码）", resource_type, target)
                print("⚠️ 检测到WAF拦截，准备重试...")
                retry_count += 1
                if retry_count <= max_retries:
                    delay = random.randint(100, 120)  # 统一延迟范围
                    print(f"⌛ 等待{delay}秒后重试...")
                    time.sleep(delay)
                continue

            if result.returncode != 0:
                log_to_file(f"请求失败（状态码：{result.returncode}", resource_type, target)
                retry_count += 1
                if retry_count <= max_retries:
                    delay = random.randint(100, 120)  # 统一延迟范围
                    print(f"⚠️ 请求失败，剩余{max_retries - retry_count}次重试（等待{delay}秒）...")
                    time.sleep(delay)
                continue

            # 保存结果
            extracted = extract_field(result.stdout, resource_type, target)
            with open(output_file, "a+", encoding="utf-8") as f:
                f.write(f"\n{'=' * 50}\n")
                f.write(f"【{target}】{resource_type} 结果（{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}）\n")
                f.write(f"{'=' * 50}\n")
                f.write("\n".join(extracted) + "\n")

            print(f"✅ 【{target}】-【{resource_type}】结果已保存（{len(extracted)}条）")
            return True

        except subprocess.TimeoutExpired:
            log_to_file("请求超时", resource_type, target)
            retry_count += 1
            if retry_count <= max_retries:
                delay = random.randint(100, 120)  # 统一延迟范围
                print(f"⚠️ 请求超时，剩余{max_retries - retry_count}次重试（等待{delay}秒）...")
                time.sleep(delay)
            continue
        except Exception as e:
            log_to_file(f"请求异常：{str(e)}", resource_type, target)
            retry_count += 1
            if retry_count <= max_retries:
                delay = random.randint(100, 120)  # 统一延迟范围
                print(f"⚠️ 请求异常，剩余{max_retries - retry_count}次重试（等待{delay}秒）...")
                time.sleep(delay)
            continue

    print(f"❌ 【{target}】-【{resource_type}】请求失败（已达最大重试次数）")
    return False


def main():
    resource_config = [("web", "web_results.txt"), ("app", "app_results.txt"), ("mapp", "mapp_results.txt")]

    try:
        file_path, targets = get_target_file_input()
        print(f"\n配置：每次请求后延迟100-120秒\n")

        # 逐个处理目标和资源类型
        for target_idx, target in enumerate(targets, 1):
            print(f"📌 开始处理第 {target_idx}/{len(targets)} 个目标：{target}")

            for res_idx, (resource_type, output_file) in enumerate(resource_config):
                run_single_request(resource_type, target, output_file)

                # 资源类型间延迟（100-120秒）
                if res_idx < len(resource_config) - 1:
                    delay = random.randint(100, 120)
                    print(f"\n⌛ 等待{delay}秒后处理下一个资源类型...")
                    time.sleep(delay)

            # 目标间延迟（100-120秒）
            if target_idx < len(targets):
                delay = random.randint(100, 120)
                print(f"\n⌛ 等待{delay}秒后处理下一个目标...\n" + "-" * 80)
                time.sleep(delay)

        print(f"\n🎉 所有目标处理完成！")
        print(f"📁 结果文件：")
        for _, output_file in resource_config:
            print(f"   - {os.path.abspath(output_file)}")
        print(f"📊 日志目录：{os.path.abspath('request_logs')}")

    except Exception as e:
        print(f"\n❌ 程序异常终止：{str(e)}")
        exit(1)


if __name__ == "__main__":
    main()

import re
import sys


def extract_pure_domains_ips(text):
    separator = r'[\r\n\s]+={50,}[\r\n\s]+'
    blocks = re.split(separator, text.strip())

    all_domains = set()
    all_ips = set()
    for block in blocks:
        block = block.strip()
        if not block:
            continue
        clean_block = re.sub(
            r'【[^】]+】web (结果（[^）]+）|未找到有效domain|响应失败（[^）]+）)',
            '',
            block
        ).strip()
        if not clean_block:
            continue
        ip_pattern = re.compile(
            r'\b(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b')
        domain_pattern = re.compile(r'\b(?:[a-zA-Z0-9-]+\.)+[a-zA-Z]{2,6}\b', re.IGNORECASE)
        ips = ip_pattern.findall(clean_block)
        if ips:
            all_ips.update(ips)

        domains = domain_pattern.findall(clean_block)
        pure_domains = [d for d in domains if not ip_pattern.match(d)]
        if pure_domains:
            all_domains.update(pure_domains)
    return sorted(all_domains), sorted(all_ips)


def main():
    if len(sys.argv) != 2:
        print("用法: python chuli_web.py <输入文件路径>")
        sys.exit(1)

    input_path = sys.argv[1]
    try:
        with open(input_path, 'rb') as f:
            text = f.read().decode('utf-8', errors='ignore').strip()
        domains, ips = extract_pure_domains_ips(text)
        print("=" * 50)
        print("纯域名列表（共{}个）".format(len(domains)))
        print("=" * 50)
        for domain in domains:
            print(domain)

        print("\n" + "=" * 50)
        print("纯IP列表（共{}个）".format(len(ips)))
        print("=" * 50)
        for ip in ips:
            print(ip)
        with open('domain.txt', 'w', encoding='utf-8') as f:
            f.write('\n'.join(domains))
        with open('ip.txt', 'w', encoding='utf-8') as f:
            f.write('\n'.join(ips))

        print(f"\n✅ 提取完成！")
        print(f"域名已保存到：domain.txt（{len(domains)}个）")
        print(f"IP已保存到：ip.txt（{len(ips)}个）")

    except FileNotFoundError:
        print(f"❌ 错误：找不到文件「{input_path}」")
    except Exception as e:
        print(f"❌ 处理出错：{str(e)}")


if __name__ == "__main__":
    main()

# 读取 domain.txt 文件并拼接数据
def read_and_concatenate(file_path):
    try:
        with open(file_path, 'r') as file:
            # 读取文件中的每一行，去掉空格和换行符
            lines = [line.strip() for line in file if line.strip()]

        # 使用 ' || ' 连接每行数据
        result = ' || '.join([f'domain="{line}" || cert="{line}"' for line in lines])

        print(result)
    except FileNotFoundError:
        print(f"文件 {file_path} 未找到。")
    except Exception as e:
        print(f"发生错误: {e}")

# 调用函数并传入文件路径
read_and_concatenate('domain.txt')

import re
import chardet


# 自动检测文件编码
def detect_encoding(filename):
    with open(filename, 'rb') as file:
        result = chardet.detect(file.read())
    return result['encoding']


# 读取文件并处理内容
def process_file(input_file, subdomain_file, url_file):
    encoding = detect_encoding(input_file)

    # 打开input.txt文件并读取每一行
    with open(input_file, 'r', encoding=encoding) as file:
        lines = file.readlines()

    # 准备两个列表来存储不同的输出内容
    subdomains = []
    urls = []

    # 定义一个正则表达式来匹配特定的IP地址URL
    specific_ip_url_pattern = re.compile(r'http[s]?://127\.0\.0\.1(:\d+)?/?')

    # 处理每一行数据
    for line in lines:
        # 去除两端的空格和换行符
        line = line.strip()

        # 检查URL是否是特定的IP地址URL
        if specific_ip_url_pattern.match(line):
            # 如果是，则直接写入url.txt
            urls.append(line)
        else:
            # 检查URL是否基于IP地址，如果是则跳过
            if re.match(r'http[s]?://\d{1,3}(\.\d{1,3}){3}(:\d+)?/?', line):
                continue  # 如果当前行是基于IP的URL，跳过不处理

            # 检查并处理每种情况
            if line.startswith('http://'):
                subdomains.append(line[7:])  # 从'http://'后面开始截取
                urls.append(line)  # 保持原样
            elif line.startswith('https://'):
                subdomains.append(line[8:])  # 从'https://'后面开始截取
                urls.append(line)  # 保持原样
            else:
                subdomains.append(line)  # 没有协议头，直接使用
                urls.append('http://' + line)  # 添加'http://'
                urls.append('https://' + line)  # 添加"https://"

    # 将处理后的数据写入相应的文件
    with open(subdomain_file, 'w', encoding=encoding) as file:
        for subdomain in subdomains:
            file.write(subdomain + '\n')

    with open(url_file, 'w', encoding=encoding) as file:
        for url in urls:
            file.write(url + '\n')


# 调用处理函数
process_file('input.txt', 'subdomain.txt', 'url.txt')

import re
import requests


def extract_unique_common_names(url, target, result_file):
    try:
        # 自定义 User-Agent 头
        headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
        }

        # 获取页面内容
        response = requests.get(url, headers=headers)
        response.raise_for_status()  # 检查是否请求成功
        html_content = response.text

        # 提取 <TD> 标签中以 .cn, .com, .org, .net 结尾的内容
        common_names = re.findall(r'<TD.*?>([^<]*\.(?:cn|com|org|net))</TD>', html_content, re.S)

        # 去重处理
        unique_common_names = set(common_names)

        # 输出提取的公用名到控制台
        if unique_common_names:
            print(f"\n===== 针对 {target} 的查询结果 =====")
            print("Extracted Unique Common Names:")
            for name in unique_common_names:
                print(name.strip())  # 去掉前后的空格

            # 仅将域名写入文件，不包含其他提示信息
            with open(result_file, "a", encoding="utf-8") as f:
                for name in unique_common_names:
                    f.write(f"{name.strip()}\n")
        else:
            print(f"\n===== 针对 {target} 的查询结果 =====")
            print("No Common Names found.")
    except requests.RequestException as e:
        print(f"\n===== 针对 {target} 的查询出错 =====")
        print(f"Error fetching URL: {e}")


def batch_query_from_file():
    # 结果文件名称
    result_file = "domain_results.txt"

    # 清空之前的结果文件
    with open(result_file, "w", encoding="utf-8") as f:
        pass  # 只打开文件并清空内容

    # 从urls.txt文件中读取域名和IP
    try:
        with open("domain.txt", "r", encoding="utf-8") as file:
            # 读取所有行并去除空行和空格
            targets = [line.strip() for line in file if line.strip()]

            if not targets:
                print("urls.txt文件中没有有效的域名或IP")
                return

            # 遍历每个目标进行查询
            for target in targets:
                # 构造查询URL
                url = f"https://crt.sh/?q={target}"
                # 调用提取函数，传入结果文件参数
                extract_unique_common_names(url, target, result_file)

    except FileNotFoundError:
        print("找不到urls.txt文件，请确保该文件存在于当前目录下")
    except Exception as e:
        print(f"处理文件时发生错误: {e}")


# 执行批量查询
batch_query_from_file()

# 打开输入文件
with open("ip.txt", "r") as input_file:
    # 读取文件内容并按行分割
    ip_addresses = input_file.read().splitlines()

# 打开输出文件
with open("output.txt", "w") as output_file:
    # 遍历每个 IP 地址
    for ip in ip_addresses:
        # 添加 http:// 前缀
        http_ip = "http://" + ip
        # 添加 https:// 前缀
        https_ip = "https://" + ip
        # 将结果写入输出文件
        output_file.write(http_ip + "\n")
        output_file.write(https_ip + "\n")