Phishing Website Detection System

# ─── Core data science libraries ───────────────────────────────────────────
import numpy as np          # numerical arrays & math
import pandas as pd         # DataFrame (table) operations
import matplotlib.pyplot as plt  # plotting
import seaborn as sns        # enhanced statistical plots

# ─── Machine learning ──────────────────────────────────────────────────────
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import (
    accuracy_score, precision_score,
    recall_score, f1_score,
    confusion_matrix, classification_report
)

# ─── Model persistence ─────────────────────────────────────────────────────
import joblib   # save and reload trained models

# ─── URL parsing (used in the live scanner, Sections 9 & 10) ───────────────
import re, urllib.parse

print('✓ All libraries imported successfully')

from google.colab import files

# This opens a file picker — select Phishing_Legitimate_full.csv from your computer
uploaded = files.upload()

# Load the CSV into a pandas DataFrame
data = pd.read_csv('Phishing_Legitimate_full.csv')

# ── Quick overview ──────────────────────────────────────────
print(f'Dataset shape:      {data.shape}')       # (rows, columns)
print(f'Phishing URLs  (1): {(data["CLASS_LABEL"]==1).sum()}')
print(f'Legitimate URLs (0): {(data["CLASS_LABEL"]==0).sum()}')
data.head()                                        # preview first 5 rows

# Column data types and non-null counts
data.info()

# Statistical summary of every column
data.describe()

# ── Class distribution visualisation ───────────────────────
fig, axes = plt.subplots(1, 2, figsize=(11, 4))

counts = data['CLASS_LABEL'].value_counts()

# Bar chart
axes[0].bar(['Legitimate (0)', 'Phishing (1)'],
            counts.values(), color=['#00e676', '#ff4d6d'])
axes[0].set_title('Class Distribution', fontweight='bold')
axes[0].set_ylabel('Number of URLs')

# Pie chart
axes[1].pie(counts.values(),
            labels=['Legitimate', 'Phishing'],
            colors=['#00e676', '#ff4d6d'],
            autopct='%1.1f%%', startangle=90)
axes[1].set_title('Class Balance', fontweight='bold')

plt.tight_layout()
plt.show()

# ── Step 1: Drop the ID column — it is not a predictive feature ──────────
data.drop('id', axis=1, inplace=True)
print(f'Columns after dropping id: {data.shape[1]}')

# ── Step 2: Check for missing values ─────────────────────────────────────
# Missing values must be handled before training — scikit-learn will throw
# an error if NaNs are present in the feature matrix.
missing = data.isnull().sum()
print('\nMissing values:')
print(missing[missing > 0] if missing.any() else '✓ None found — dataset is clean')

# ── Step 3: Sanity check on max values ───────────────────────────────────
# Most features should be binary (0 or 1). Any unexpectedly large max
# value might indicate a data entry error or an outlier to investigate.
print('\nMax value per column (binary features should max at 1):')
print(data.max(axis=0))

# Compute absolute correlation of every feature with the class label
# We use .abs() because a strong negative correlation is just as useful
# as a strong positive one — it still means the feature is informative.
corr = data.corr()['CLASS_LABEL'].abs().sort_values(ascending=False)

# Plot top 20 (skip index 0 which is CLASS_LABEL itself)
plt.figure(figsize=(10, 6))
corr[1:21].plot(kind='barh', color='#00d4ff')
plt.title('Top 20 Features by Absolute Correlation with CLASS_LABEL', fontweight='bold')
plt.xlabel('Absolute Correlation Coefficient')
plt.tight_layout()
plt.show()

# Print top 10 numerically
print('\nTop 10 correlated features:\n')
print(corr[1:11].to_string())

# ── Separate features from target ─────────────────────────────────────────
X = data.drop('CLASS_LABEL', axis=1)   # all columns except the label
y = data['CLASS_LABEL']                 # only the label column

# Save the column names — we need this later when scanning live URLs
# to ensure we build the feature vector in the exact same order the model expects
FEATURE_NAMES = X.columns.tolist()

print(f'Feature matrix X: {X.shape}')        # should be (11055, 48)
print(f'Target vector y:  {y.shape}')        # should be (11055,)
print(f'Number of features: {len(FEATURE_NAMES)}')

# ── 80 / 20 train-test split ──────────────────────────────────────────────
X_train, X_test, y_train, y_test = train_test_split(
    X, y,
    test_size=0.20,       # 20% held out for testing
    random_state=42,      # reproducible split
    stratify=y             # preserve class balance in both halves
)

print(f'\nTraining samples: {X_train.shape[0]}')
print(f'Test samples:     {X_test.shape[0]}')
print(f'\nTraining label split (should be ~50/50):\n{y_train.value_counts()}')

from sklearn.ensemble import RandomForestClassifier
import joblib

# ── Initialise the model ──────────────────────────────────────────────────
rfc = RandomForestClassifier(
    n_estimators=100,     # build 100 individual decision trees
    random_state=42,      # reproducible results
    n_jobs=-1             # use all CPU cores (speeds up training)
)

# ── Train the model ───────────────────────────────────────────────────────
# .fit() is where all the learning happens. This may take 5-20 seconds.
print('⏳ Training Random Forest on 80% of the data...')
rfc.fit(X_train, y_train)
print('✓ Training complete!')

# ── Generate predictions on the test set ──────────────────────────────────
# preds gives the hard label (0 or 1)
# pred_proba gives the probability of phishing (0.0 to 1.0) — useful for tuning
preds      = rfc.predict(X_test)
pred_proba = rfc.predict_proba(X_test)[:, 1]   # column 1 = probability of class 1 (phishing)

# ── Save the trained model to disk ────────────────────────────────────────
# Download this file from the Colab Files panel (left sidebar → folder icon)
# to reuse it without retraining.
joblib.dump(rfc, 'phishing_rf_model.pkl')
print('💾 Model saved as phishing_rf_model.pkl')

from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score
from sklearn.metrics import classification_report, confusion_matrix
import seaborn as sns

# ── Compute the four core metrics ──────────────────────────────────────────
acc  = accuracy_score(y_test, preds)
prec = precision_score(y_test, preds)
rec  = recall_score(y_test, preds)
f1   = f1_score(y_test, preds)

print('═' * 45)
print(f'  Accuracy:  {acc:.4f}  ({acc*100:.2f}%)')
print(f'  Precision: {prec:.4f}  — of flagged URLs, this many are real phishing')
print(f'  Recall:    {rec:.4f}  — of all phishing URLs, this many were caught')
print(f'  F1-Score:  {f1:.4f}  — balanced precision/recall score')
print('═' * 45)

# ── Full per-class breakdown ───────────────────────────────────────────────
print('\n📊 Classification Report:')
print(classification_report(y_test, preds,
                             target_names=['Legitimate', 'Phishing']))

# ── Confusion matrix heatmap ───────────────────────────────────────────────
# Each cell shows the count: rows = actual class, columns = predicted class
cm = confusion_matrix(y_test, preds)

plt.figure(figsize=(6, 5))
sns.heatmap(cm, annot=True, fmt='d', cmap='Blues',
            xticklabels=['Predicted: Legit', 'Predicted: Phishing'],
            yticklabels=['Actual: Legit', 'Actual: Phishing'])
plt.title('Confusion Matrix — Phishing Detector', fontweight='bold')
plt.tight_layout()
plt.show()

# ── Explain the confusion matrix cells ────────────────────────────────────
tn, fp, fn, tp = cm.ravel()
print(f'\nTrue Negatives  (correctly marked safe):     {tn}')
print(f'False Positives (safe URLs wrongly flagged): {fp}  ← false alarms')
print(f'False Negatives (phishing URLs missed!):     {fn}  ← dangerous misses')
print(f'True Positives  (phishing correctly caught):  {tp}')

import pandas as pd

# ── Build a ranked DataFrame of all feature importances ───────────────────
feat_df = pd.DataFrame({
    'feature':    FEATURE_NAMES,
    'importance': rfc.feature_importances_   # built-in Random Forest attribute
}).sort_values('importance', ascending=False).reset_index(drop=True)

# ── Visualise the top 20 ──────────────────────────────────────────────────
top20  = feat_df.head(20)
# Highlight the top 5 in bright cyan; the rest in a darker shade
colors = ['#00d4ff' if i < 5 else '#2a6080' for i in range(20)]

plt.figure(figsize=(11, 7))
plt.barh(top20['feature'][::-1],
         top20['importance'][::-1],
         color=colors[::-1])
plt.title('Top 20 Most Important Features for Phishing Detection',
          fontweight='bold', fontsize=13)
plt.xlabel('Importance Score (contribution to predictions across all trees)')
plt.tight_layout()
plt.show()

# ── Print the top 10 as a table ───────────────────────────────────────────
print('\nTop 10 most important features:\n')
print(feat_df.head(10).to_string(index=False))

import re, urllib.parse

def extract_url_features(url):
    """
    Extract phishing-detection features from a raw URL string.
    Returns a dict of feature name → value, aligned to the training
    dataset's column order so the model can read it correctly.
    HTML-based features are defaulted to 0 (URL-only mode).
    """

    try:
        parsed = urllib.parse.urlparse(url)
    except Exception:
        return {}

    # Split URL into its components for targeted analysis
    domain = parsed.netloc.replace('www.', '')   # hostname without www
    path   = parsed.path                            # everything after the domain
    query  = parsed.query                           # everything after the ?

    # Known brand names — phishing sites often embed these in subdomains/paths
    BRAND_WORDS = ['paypal','ebay','google','amazon','microsoft','apple','chase']
    # Sensitive keywords that phishing URLs commonly contain
    SENS_WORDS  = ['secure','account','update','login','verify','banking','password']

    f = {}

    # ── URL structure features ─────────────────────────────────────────────
    f['NumDots']           = url.count('.')              # total dots in full URL
    f['SubdomainLevel']    = max(0, domain.count('.') - 1)  # subdomain depth (0 = none)
    f['PathLevel']         = len([p for p in path.split('/') if p])
    f['UrlLength']         = len(url)                     # total URL character length
    f['NumDash']           = url.count('-')              # dashes anywhere in URL
    f['NumDashInHostname'] = domain.count('-')           # dashes only in domain
    f['AtSymbol']          = 1 if '@' in url else 0     # @ tricks browsers into ignoring the domain
    f['TildeSymbol']       = 1 if '~' in url else 0     # tilde often signals old CGI phishing pages
    f['NumUnderscore']     = url.count('_')
    f['NumPercent']        = url.count('%')              # URL-encoded chars — used to obfuscate
    f['NumQueryComponents'] = len(urllib.parse.parse_qs(query))
    f['NumAmpersand']      = url.count('&')
    f['NumHash']           = url.count('#')
    f['NumNumericChars']   = sum(c.isdigit() for c in url)
    f['NoHttps']           = 0 if parsed.scheme == 'https' else 1  # 1 = no HTTPS (suspicious)
    f['RandomString']      = 1 if re.search(r'[a-z]{10,}', domain) else 0
    f['IpAddress']         = 1 if re.match(r'\d+\.\d+\.\d+\.\d+', domain) else 0  # IP instead of domain name
    f['DomainInSubdomains'] = 1 if any(b in domain for b in BRAND_WORDS) else 0
    f['DomainInPaths']     = 1 if any(b in path for b in BRAND_WORDS) else 0
    f['HttpsInHostname']   = 1 if 'https' in domain else 0   # e.g. "https-paypal.com" — deceptive
    f['HostnameLength']    = len(domain)
    f['PathLength']        = len(path)
    f['QueryLength']       = len(query)
    f['DoubleSlashInPath'] = 1 if '//' in path else 0
    f['NumSensitiveWords'] = sum(1 for w in SENS_WORDS if w in url.lower())
    f['UrlLengthRT']      = 1 if len(url) > 75 else 0   # binary: is URL suspiciously long?
    f['SubdomainLevelRT']  = f['SubdomainLevel']          # same value, used as a separate feature

    # ── HTML-based features — default to 0 in URL-only mode ───────────────
    # These features require fetching and parsing the webpage's HTML source.
    # Since we only have the URL here, we set them to 0 (neutral).
    # The model still works well — it just can't use these signals.
    for col in ['PctExtHyperlinks','PctExtResourceUrls','ExtFavicon',
                 'InsecureForms','RelativeFormAction','ExtFormAction',
                 'AbnormalFormAction','PctNullSelfRedirectHyperlinks',
                 'FrequentDomainNameMismatch','FakeLinkInStatusBar',
                 'RightClickDisabled','PopUpWindow','SubmitInfoToEmail',
                 'IframeOrFrame','MissingTitle','ImagesOnlyInForm',
                 'PctExtNullSelfRedirectHyperlinksRT']:
        f[col] = 0

    return f

print('✓ extract_url_features() is ready')

import joblib
import numpy as np

# Load the model we saved in Section 6
# If Colab restarted, retrain first (run Sections 2–6 again)
model = joblib.load('phishing_rf_model.pkl')

def scan_url(url, verbose=True):
    """
    Scan a single URL using the trained Random Forest model.
    
    Parameters:
        url     : the full URL string (include https:// or http://)
        verbose : if True, prints a human-readable verdict with feature details
    
    Returns:
        (prediction, confidence)
        prediction  : 0 = Legitimate, 1 = Phishing
        confidence  : probability of being phishing (0.0 to 1.0)
    """
    # Extract features from the URL string
    features = extract_url_features(url)

    # Build the feature vector in the EXACT column order the model expects
    vec = np.array([features.get(col, 0) for col in FEATURE_NAMES]).reshape(1, -1)

    # Get the hard prediction (0 or 1) and the phishing probability
    pred = model.predict(vec)[0]
    conf = model.predict_proba(vec)[0][1]   # probability of class 1 (phishing)

    if verbose:
        verdict = '🚨 PHISHING' if pred == 1 else '✓  LEGITIMATE'
        print(f'\n{"─"*55}')
        print(f'URL      : {url}')
        print(f'Verdict  : {verdict}')
        print(f'Confidence (phishing probability): {conf*100:.1f}%')
        print(f'\nKey features that influenced this result:')
        print(f'  HTTPS:            {"Yes ✓" if not features["NoHttps"] else "NO — suspicious ⚠️"}')
        print(f'  URL length:       {features["UrlLength"]} chars')
        print(f'  Domain dashes:    {features["NumDashInHostname"]}')
        print(f'  Subdomain depth:  {features["SubdomainLevel"]} (0 = no subdomains)')
        print(f'  Sensitive words:  {features["NumSensitiveWords"]}')
        print(f'  IP as domain:     {"YES — suspicious ⚠️" if features["IpAddress"] else "No ✓"}')
        print(f'  Brand in subdomain: {"YES — suspicious ⚠️" if features["DomainInSubdomains"] else "No ✓"}')
        print(f'{"─"*55}')

    return pred, conf

# ── Test the scanner on two example URLs ─────────────────────────────────
scan_url('https://www.google.com')
scan_url('http://secure-paypal-verify.suspicious-site.com/login')

import pandas as pd

# ── Define your list of URLs to scan ──────────────────────────────────────
# Add your own URLs here, or read from a file (see tip below)
urls_to_scan = [
    'https://www.google.com',
    'https://www.amazon.com',
    'http://secure-paypa1-login.verify-account.com/login',  # note typo 'paypa1' — common trick
    'http://192.168.1.1/update-credentials',              # raw IP address — suspicious
    'https://github.com',
    'http://microsoft-account-verify.tk/signin',          # .tk domain, suspicious keywords
    'https://www.wikipedia.org',
    'http://banking-update.secure-login.verify.net/account', # stacked sensitive words
]

# ── Scan each URL and collect results ────────────────────────────────────
results = []
print(f"{'#':<4} {'VERDICT':<14} {'CONF':>6}  URL")
print('─' * 80)

for i, url in enumerate(urls_to_scan, 1):
    try:
        pred, conf = scan_url(url, verbose=False)   # silent mode for batch
        verdict = 'PHISHING' if pred == 1 else 'LEGITIMATE'
        icon    = '🚨' if pred == 1 else '✓'
        results.append({
            'url':        url,
            'verdict':    verdict,
            'confidence': round(conf, 4)
        })
        print(f"{i:<4} {icon} {verdict:<12} {conf*100:>5.1f}%  {url[:55]}")
    except Exception as e:
        # Don't crash on a single bad URL — log the error and move on
        print(f"{i:<4} ⚠️  ERROR        {'N/A':>6}  {str(e)[:40]}")

# ── Summary statistics ────────────────────────────────────────────────────
ph  = sum(1 for r in results if r['verdict'] == 'PHISHING')
leg = len(results) - ph
print(f'\n📊 Summary: {leg} Legitimate  |  {ph} Phishing  |  {len(results)} total scanned')

# ── Export results to CSV ─────────────────────────────────────────────────
# Download this from the Colab Files panel for your report
pd.DataFrame(results).to_csv('scan_results.csv', index=False)
print('💾 Results saved to scan_results.csv')