import joblib import pandas as pd import re import time import math import psutil import os from collections import Counter from urllib.parse import urlparse # ============================================================================= # CONSTANTS (Synchronized with Training) # ============================================================================= PHISHING_KEYWORDS = [ "login", "verify", "update", "secure", "account", "bank", "confirm", "signin", "password", "reset", "admin" ] SUSPICIOUS_TLDS = ["xyz", "top", "club", "tk", "ml", "ga", "cf", "buzz", "cn", "ru"] BRANDS = ["paypal", "google", "facebook", "bank", "apple", "microsoft", "amazon"] SHORTENERS = ["bit.ly", "tinyurl.com", "t.co", "goo.gl", "is.gd", "ow.ly", "buff.ly"] # ----------------------------------------------------------------------------- # WHITELIST — Only include domains that: # (a) Never host user-generated / third-party content, AND # (b) Are not abused as phishing infrastructure # # REMOVED google.com — docs/drive/sites/forms are actively abused for phishing # REMOVED facebook.com — fb.com links frequently used in phishing redirects # github.com — kept; raw hosting is rare for credential phishing # youtube.com — kept; no login-harvesting vectors via URL alone # ----------------------------------------------------------------------------- TRUSTED_DOMAINS = [ # "google.com", # DISABLED: docs/drive/sites subdomains host real phishing # "facebook.com", # DISABLED: used as phishing redirect infrastructure "github.com", "youtube.com", "microsoft.com", "apple.com", ] # Brand domain mapping — used to distinguish impersonation from genuine brand URLs BRAND_REAL_DOMAINS = { "paypal": ["paypal.com"], "google": ["google.com", "google.co"], "facebook": ["facebook.com", "fb.com"], "bank": [], # Generic word — flag all uses in domain "apple": ["apple.com"], "microsoft": ["microsoft.com", "live.com", "outlook.com"], "amazon": ["amazon.com", "aws.amazon.com"], } # ============================================================================= # HELPER FUNCTIONS # ============================================================================= def is_ip(domain: str) -> bool: return bool(re.match(r"^\d{1,3}(\.\d{1,3}){3}$", domain)) def entropy(s: str) -> float: if not s: return 0.0 prob = [n / len(s) for _, n in Counter(s).items()] return -sum(p * math.log2(p) for p in prob) def has_brand_impersonation(url_lower: str, domain: str) -> int: """ Returns 1 only when a brand keyword appears in the URL but the domain is NOT the brand's real domain. This eliminates false positives on legitimate brand pages while still catching brand-spoofing phishing sites. FIX: Previously `any(b in url.lower() for b in BRANDS)` flagged every URL that merely mentioned a brand — including the brand's own real site. """ for brand, real_domains in BRAND_REAL_DOMAINS.items(): if brand in url_lower: # If the domain IS one of the real brand domains, it is not impersonation if any(domain == rd or domain.endswith("." + rd) for rd in real_domains): continue return 1 return 0 def count_domain_phishing_keywords(domain: str) -> int: """ Count phishing keywords that appear in the DOMAIN only (not the path/query). FIX: Checking the full URL caused false positives — legitimate sites use words like 'login', 'secure', 'account' in their paths all the time. Phishing sites tend to embed these words in the domain name itself (e.g. 'paypal-secure-update.com'), which is a much stronger signal. """ return sum(k in domain for k in PHISHING_KEYWORDS) def count_path_phishing_keywords(path: str) -> int: """ Separate, lower-weight count of phishing keywords found in the path. The model can assign a lower coefficient to this than to domain keywords. """ return sum(k in path for k in PHISHING_KEYWORDS) def is_trusted_domain(domain: str, path: str) -> bool: """ Whitelist check with path guard. FIX: A bare `endswith(trusted)` check was passing phishing URLs hosted on Google/Facebook infrastructure straight through as 'LEGITIMATE (Whitelisted)'. Now we also verify the path contains no phishing keywords before trusting. """ for trusted in TRUSTED_DOMAINS: if domain == trusted or domain.endswith("." + trusted): # Extra guard: if the path itself looks phishy, don't whitelist if count_path_phishing_keywords(path) == 0: return True return False # ============================================================================= # FEATURE EXTRACTION # ============================================================================= def extract_features(url: str) -> dict: url = url.strip() # Normalise scheme so urlparse works correctly (matches training-time behaviour) if not url.startswith(("http://", "https://")): url = "https://" + url parsed = urlparse(url) domain = parsed.netloc.lower().split(":")[0] path = parsed.path.lower() query = parsed.query.lower() length = len(url) num_digits = sum(c.isdigit() for c in url) num_special = sum(not c.isalnum() for c in url) tld = domain.split(".")[-1] if "." in domain else "" return { # ── Length / size signals ────────────────────────────────────────── "url_length": length, "domain_length": len(domain), "path_length": len(path), # ── Structural signals ───────────────────────────────────────────── "num_subdomains": domain.count("."), "num_hyphens_url": url.count("-"), "num_hyphens_domain": domain.count("-"), # ── Character-level signals ──────────────────────────────────────── "num_digits": num_digits, "digit_ratio": num_digits / length if length > 0 else 0, "num_special_chars": num_special, "special_char_ratio": num_special / length if length > 0 else 0, # ── Query signals ────────────────────────────────────────────────── "num_query_params": query.count("&") + (1 if query else 0), # ── Keyword signals (split by location) ─────────────────────────── # FIX: domain keywords are a far stronger phishing signal than path keywords "num_phishing_keywords_domain": count_domain_phishing_keywords(domain), "num_phishing_keywords_path": count_path_phishing_keywords(path), # ── Protocol signals ─────────────────────────────────────────────── # FIX: Removed is_https as a positive signal — 57 % of phishing URLs # in the training set already use HTTPS, so rewarding HTTPS inflates # false-negative rate. We flag plain HTTP as a weak bad signal instead. "is_http_only": int(url.startswith("http://") and not url.startswith("https://")), "https_in_domain": int("https" in domain), # e.g. 'httpsservices.evil.ru' # ── Domain-type signals ──────────────────────────────────────────── "has_www": int("www." in domain), "tld_length": len(tld), "is_ip_address": int(is_ip(domain)), "is_suspicious_tld": int(tld in SUSPICIOUS_TLDS), # ── Redirect / obfuscation signals ──────────────────────────────── "is_shortener": int(any(s in domain for s in SHORTENERS)), "has_at_symbol": int("@" in url), "has_double_slash_path": int("//" in path), # ── Brand impersonation signal ───────────────────────────────────── # FIX: Previously flagged any URL mentioning a brand name, including # the brand's own legitimate website. Now only flags when the # brand word appears on a domain that is NOT the brand's real domain. "has_brand_name": has_brand_impersonation(url.lower(), domain), # ── Entropy (randomness) ─────────────────────────────────────────── "domain_entropy": entropy(domain), } # ============================================================================= # LOAD MODEL # ============================================================================= load_start = time.perf_counter() # Ensure these files exist in the same directory and were produced by a # training script that uses the SAME feature set defined above. model = joblib.load("phishing_model.pkl") FEATURES = joblib.load("features.pkl") load_end = time.perf_counter() print(f"Model loading time: {load_end - load_start:.6f} seconds") # ============================================================================= # PREDICTION FUNCTION # ============================================================================= def predict_url(url: str, threshold: float = 0.5): timings = {} url = url.strip() # Normalise for domain extraction url_to_parse = url if url.startswith(("http://", "https://")) else "https://" + url parsed = urlparse(url_to_parse) domain = parsed.netloc.lower().split(":")[0] path = parsed.path.lower() # ── Whitelist check (with path guard) ───────────────────────────────── if is_trusted_domain(domain, path): return "LEGITIMATE (Whitelisted)", 0.00, {} # ── Feature extraction ───────────────────────────────────────────────── feat_start = time.perf_counter() features = extract_features(url) # Debug: find missing keys missing = [f for f in FEATURES if f not in features] if missing: print(f"Missing features: {missing}") # Fill missing with 0 as fallback for key in missing: features[key] = 0 X = pd.DataFrame([features])[FEATURES] feat_end = time.perf_counter() timings["feature_extraction"] = feat_end - feat_start # ── Model inference ──────────────────────────────────────────────────── pred_start = time.perf_counter() prob = model.predict_proba(X)[0][1] pred_end = time.perf_counter() timings["prediction"] = pred_end - pred_start label = "PHISHING" if prob >= threshold else "LEGITIMATE" return label, prob, timings # ============================================================================= # TESTING & PROFILING # ============================================================================= test_urls = [ # Should be LEGITIMATE "https://www.tradingview.com", "www.tradingview.com", "https://www.reynoldstransfer.com/", # Should be PHISHING — google-hosted phishing (was whitelisted before fix) "https://docs.google.com/forms/d/e/1FAIpQLSel0WzlUSRy5Yq7D5GuYeKTGE6eCIY12-8pYpvhFYihnkOQ8g/viewform", "https://sites.google.com/view/paypal-customer-services/", # Should be PHISHING — classic patterns "paypal-secure-update.com/login", "bit.ly/3xYz8q", "https://httpsservices.runescape.com-ov.ru/", ] for url in test_urls: total_start = time.perf_counter() process = psutil.Process(os.getpid()) mem_before = process.memory_info().rss / (1024 * 1024) label, prob, timings = predict_url(url) mem_after = process.memory_info().rss / (1024 * 1024) total_end = time.perf_counter() print("\n" + "=" * 55) print(f"URL : {url}") print(f"Prediction: {label}") print(f"Phishing probability: {prob:.3f}") if timings: print(f"Feature extraction : {timings.get('feature_extraction', 0):.6f}s") print(f"Model inference : {timings.get('prediction', 0):.6f}s") else: print("Skipped ML inference (Whitelisted)") print(f"Total inference time : {total_end - total_start:.6f}s") print(f"Memory fluctuation : {mem_after - mem_before:.2f} MB")