Hoghoghi / frontend /scraping_dashboard.html
Really-amin's picture
Upload 143 files
c636ebf verified
raw
history blame
36.1 kB
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Legal Dashboard - Scraping &amp; Rating System</title>
<link href="https://cdn.jsdelivr.net/npm/[email protected]/dist/css/bootstrap.min.css" rel="stylesheet">
<link href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.4.0/css/all.min.css" rel="stylesheet">
<link href="https://cdn.jsdelivr.net/npm/[email protected]/dist/chart.min.css" rel="stylesheet">
<!-- Load API Client and Core System -->
<script src="js/api-client.js"></script>
<script src="js/core.js"></script>
<script src="js/notifications.js"></script>
<script src="js/scraping-control.js"></script>
<style>
:root {
--primary-color: #2c3e50;
--secondary-color: #3498db;
--success-color: #27ae60;
--warning-color: #f39c12;
--danger-color: #e74c3c;
--light-bg: #f8f9fa;
--dark-bg: #343a40;
}
body {
font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
min-height: 100vh;
}
.navbar-custom {
background: linear-gradient(135deg, var(--primary-color), var(--secondary-color));
}
.navbar-brand {
font-weight: bold;
font-size: 1.5rem;
}
.card {
border: none;
border-radius: 15px;
box-shadow: 0 10px 30px rgba(0,0,0,0.1);
transition: transform 0.3s ease;
}
.card:hover {
transform: translateY(-5px);
}
.card-header {
background: linear-gradient(135deg, var(--primary-color), var(--secondary-color));
color: white;
border-radius: 15px 15px 0 0 !important;
font-weight: bold;
}
.btn-primary {
background: linear-gradient(135deg, var(--secondary-color), var(--primary-color));
border: none;
border-radius: 25px;
padding: 10px 25px;
font-weight: bold;
}
.btn-success {
background: linear-gradient(135deg, var(--success-color), #2ecc71);
border: none;
border-radius: 25px;
}
.btn-warning {
background: linear-gradient(135deg, var(--warning-color), #f1c40f);
border: none;
border-radius: 25px;
}
.btn-danger {
background: linear-gradient(135deg, var(--danger-color), #c0392b);
border: none;
border-radius: 25px;
}
.progress {
height: 25px;
border-radius: 15px;
background-color: #e9ecef;
}
.progress-bar {
border-radius: 15px;
font-weight: bold;
}
.stats-card {
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
color: white;
border-radius: 15px;
padding: 20px;
margin-bottom: 20px;
}
.stats-number {
font-size: 2.5rem;
font-weight: bold;
}
.stats-label {
font-size: 0.9rem;
opacity: 0.8;
}
.table {
border-radius: 10px;
overflow: hidden;
}
.table thead th {
background: linear-gradient(135deg, var(--primary-color), var(--secondary-color));
color: white;
border: none;
font-weight: bold;
}
.badge {
border-radius: 20px;
padding: 8px 15px;
font-weight: bold;
}
.alert {
border-radius: 15px;
border: none;
}
.form-control, .form-select {
border-radius: 10px;
border: 2px solid #e9ecef;
transition: border-color 0.3s ease;
}
.form-control:focus, .form-select:focus {
border-color: var(--secondary-color);
box-shadow: 0 0 0 0.2rem rgba(52, 152, 219, 0.25);
}
.modal-content {
border-radius: 15px;
border: none;
}
.modal-header {
background: linear-gradient(135deg, var(--primary-color), var(--secondary-color));
color: white;
border-radius: 15px 15px 0 0;
}
.loading-spinner {
display: inline-block;
width: 20px;
height: 20px;
border: 3px solid #f3f3f3;
border-top: 3px solid var(--secondary-color);
border-radius: 50%;
animation: spin 1s linear infinite;
}
@keyframes spin {
0% { transform: rotate(0deg); }
100% { transform: rotate(360deg); }
}
.notification {
position: fixed;
top: 20px;
right: 20px;
z-index: 9999;
border-radius: 10px;
padding: 15px 20px;
color: white;
font-weight: bold;
transform: translateX(400px);
transition: transform 0.3s ease;
}
.notification.show {
transform: translateX(0);
}
.notification.success {
background: linear-gradient(135deg, var(--success-color), #2ecc71);
}
.notification.warning {
background: linear-gradient(135deg, var(--warning-color), #f1c40f);
}
.notification.error {
background: linear-gradient(135deg, var(--danger-color), #c0392b);
}
.chart-container {
position: relative;
height: 300px;
margin: 20px 0;
}
.job-status {
padding: 10px 15px;
border-radius: 20px;
font-weight: bold;
font-size: 0.9rem;
}
.status-pending { background-color: #f8f9fa; color: #6c757d; }
.status-processing { background-color: #fff3cd; color: #856404; }
.status-completed { background-color: #d1ecf1; color: #0c5460; }
.status-failed { background-color: #f8d7da; color: #721c24; }
.rating-badge {
padding: 5px 10px;
border-radius: 15px;
font-weight: bold;
font-size: 0.8rem;
}
.rating-excellent { background-color: #d4edda; color: #155724; }
.rating-good { background-color: #d1ecf1; color: #0c5460; }
.rating-average { background-color: #fff3cd; color: #856404; }
.rating-poor { background-color: #f8d7da; color: #721c24; }
.rating-unrated { background-color: #e2e3e5; color: #383d41; }
</style>
</head>
<body>
<!-- Navigation -->
<nav class="navbar navbar-expand-lg navbar-dark navbar-custom">
<div class="container-fluid">
<a class="navbar-brand" href="#">
<i class="fas fa-spider me-2"></i>
Legal Dashboard - Scraping &amp; Rating
</a>
<div class="navbar-nav ms-auto">
<a class="nav-link" href="#" onclick="showNotification('System is running smoothly', 'success')">
<i class="fas fa-heartbeat me-1"></i>
System Health
</a>
</div>
</div>
</nav>
<div class="container-fluid mt-4">
<!-- Statistics Cards -->
<div class="row mb-4">
<div class="col-md-3">
<div class="stats-card text-center">
<div class="stats-number" id="totalItems">0</div>
<div class="stats-label">Total Items Scraped</div>
</div>
</div>
<div class="col-md-3">
<div class="stats-card text-center">
<div class="stats-number" id="activeJobs">0</div>
<div class="stats-label">Active Jobs</div>
</div>
</div>
<div class="col-md-3">
<div class="stats-card text-center">
<div class="stats-number" id="avgRating">0.0</div>
<div class="stats-label">Average Rating</div>
</div>
</div>
<div class="col-md-3">
<div class="stats-card text-center">
<div class="stats-number" id="totalRated">0</div>
<div class="stats-label">Items Rated</div>
</div>
</div>
</div>
<div class="row">
<!-- Scraping Control Panel -->
<div class="col-lg-4">
<div class="card mb-4">
<div class="card-header">
<i class="fas fa-spider me-2"></i>
Scraping Control Panel
</div>
<div class="card-body">
<form id="scrapingForm">
<div class="mb-3">
<label for="urls" class="form-label">URLs to Scrape</label>
<textarea class="form-control" id="urls" rows="4" placeholder="Enter URLs (one per line)&#10;Example:&#10;https://example.com/page1&#10;https://example.com/page2"></textarea>
</div>
<div class="mb-3">
<label for="strategy" class="form-label">Scraping Strategy</label>
<select class="form-select" id="strategy">
<option value="general">General</option>
<option value="legal_documents">Legal Documents</option>
<option value="news_articles">News Articles</option>
<option value="academic_papers">Academic Papers</option>
<option value="government_sites">Government Sites</option>
<option value="custom">Custom</option>
</select>
</div>
<div class="mb-3">
<label for="keywords" class="form-label">Keywords (optional)</label>
<input type="text" class="form-control" id="keywords" placeholder="Enter keywords separated by commas">
</div>
<div class="row">
<div class="col-md-6">
<div class="mb-3">
<label for="maxDepth" class="form-label">Max Depth</label>
<input type="number" class="form-control" id="maxDepth" value="1" min="1" max="5">
</div>
</div>
<div class="col-md-6">
<div class="mb-3">
<label for="delay" class="form-label">Delay (seconds)</label>
<input type="number" class="form-control" id="delay" value="1.0" min="0.1" max="10.0" step="0.1">
</div>
</div>
</div>
<button type="submit" class="btn btn-primary w-100" id="startScrapingBtn">
<i class="fas fa-play me-2"></i>
Start Scraping Job
</button>
</form>
</div>
</div>
<!-- Rating Controls -->
<div class="card mb-4">
<div class="card-header">
<i class="fas fa-star me-2"></i>
Rating Controls
</div>
<div class="card-body">
<button type="button" class="btn btn-success w-100 mb-2" onclick="rateAllItems()">
<i class="fas fa-star me-2"></i>
Rate All Unrated Items
</button>
<button type="button" class="btn btn-warning w-100 mb-2" onclick="getLowQualityItems()">
<i class="fas fa-exclamation-triangle me-2"></i>
Get Low Quality Items
</button>
<button type="button" class="btn btn-info w-100" onclick="refreshStatistics()">
<i class="fas fa-sync-alt me-2"></i>
Refresh Statistics
</button>
</div>
</div>
</div>
<!-- Active Jobs -->
<div class="col-lg-8">
<div class="card mb-4">
<div class="card-header d-flex justify-content-between align-items-center">
<span><i class="fas fa-tasks me-2"></i>Active Scraping Jobs</span>
<button type="button" class="btn btn-sm btn-outline-light" onclick="refreshJobs()" aria-label="Refresh jobs">
<i class="fas fa-sync-alt"></i>
</button>
</div>
<div class="card-body">
<div id="jobsContainer">
<div class="text-center text-muted">
<i class="fas fa-spinner fa-spin fa-2x mb-2"></i>
<p>Loading jobs...</p>
</div>
</div>
</div>
</div>
<!-- Charts -->
<div class="row">
<div class="col-md-6">
<div class="card">
<div class="card-header">
<i class="fas fa-chart-pie me-2"></i>
Rating Distribution
</div>
<div class="card-body">
<div class="chart-container">
<canvas id="ratingChart"></canvas>
</div>
</div>
</div>
</div>
<div class="col-md-6">
<div class="card">
<div class="card-header">
<i class="fas fa-chart-bar me-2"></i>
Language Distribution
</div>
<div class="card-body">
<div class="chart-container">
<canvas id="languageChart"></canvas>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
<!-- Scraped Items Table -->
<div class="row mt-4">
<div class="col-12">
<div class="card">
<div class="card-header d-flex justify-content-between align-items-center">
<span><i class="fas fa-list me-2"></i>Scraped Items</span>
<div>
<button type="button" class="btn btn-sm btn-outline-light me-2" onclick="refreshItems()" aria-label="Refresh items">
<i class="fas fa-sync-alt"></i>
</button>
<select class="form-select form-select-sm d-inline-block w-auto" id="itemFilter">
<option value="">All Items</option>
<option value="completed">Completed</option>
<option value="failed">Failed</option>
<option value="rated">Rated</option>
</select>
</div>
</div>
<div class="card-body">
<div class="table-responsive">
<table class="table table-hover">
<thead>
<tr>
<th scope="col">Title</th>
<th scope="col">URL</th>
<th scope="col">Status</th>
<th scope="col">Rating</th>
<th scope="col">Language</th>
<th scope="col">Word Count</th>
<th scope="col">Actions</th>
</tr>
</thead>
<tbody id="itemsTableBody">
<tr>
<td colspan="7" class="text-center text-muted">
<i class="fas fa-spinner fa-spin me-2"></i>
Loading items...
</td>
</tr>
</tbody>
</table>
</div>
</div>
</div>
</div>
</div>
</div>
<!-- Notification Container -->
<div id="notificationContainer"></div>
<!-- Scripts -->
<script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/js/bootstrap.bundle.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/chart.min.js"></script>
<script>
// Global variables
let ratingChart = null;
let languageChart = null;
let refreshInterval = null;
// Initialize dashboard
document.addEventListener('DOMContentLoaded', function() {
loadDashboard();
startAutoRefresh();
});
// Load dashboard data
async function loadDashboard() {
try {
await Promise.all([
loadStatistics(),
loadJobs(),
loadItems(),
loadCharts()
]);
} catch (error) {
console.error('Error loading dashboard:', error);
showNotification('Error loading dashboard data', 'error');
}
}
// Load statistics
async function loadStatistics() {
try {
const [scrapingStats, ratingSummary] = await Promise.all([
fetch('/api/scrape/statistics').then(r => r.json()),
fetch('/api/rating/summary').then(r => r.json())
]);
document.getElementById('totalItems').textContent = scrapingStats.total_items || 0;
document.getElementById('activeJobs').textContent = scrapingStats.active_jobs || 0;
document.getElementById('avgRating').textContent = (ratingSummary.average_score || 0).toFixed(2);
document.getElementById('totalRated').textContent = ratingSummary.total_rated || 0;
} catch (error) {
console.error('Error loading statistics:', error);
}
}
// Load jobs
async function loadJobs() {
try {
const response = await fetch('/api/scrape/status');
const jobs = await response.json();
const container = document.getElementById('jobsContainer');
if (jobs.length === 0) {
container.innerHTML = '<div class="text-center text-muted"><p>No active jobs</p></div>';
return;
}
let html = '';
jobs.forEach(job => {
const progress = job.progress * 100;
html += `
<div class="card mb-3">
<div class="card-body">
<div class="d-flex justify-content-between align-items-center mb-2">
<h6 class="card-title mb-0">Job ${job.job_id}</h6>
<span class="job-status status-${job.status}">${job.status}</span>
</div>
<div class="progress mb-2">
<div class="progress-bar" role="progressbar" style="width: ${progress}%"
aria-valuenow="${progress}" aria-valuemin="0" aria-valuemax="100">
${progress.toFixed(1)}%
</div>
</div>
<div class="row text-center">
<div class="col-4">
<small class="text-muted">Total</small><br>
<strong>${job.total_items}</strong>
</div>
<div class="col-4">
<small class="text-muted">Completed</small><br>
<strong class="text-success">${job.completed_items}</strong>
</div>
<div class="col-4">
<small class="text-muted">Failed</small><br>
<strong class="text-danger">${job.failed_items}</strong>
</div>
</div>
</div>
</div>
`;
});
container.innerHTML = html;
} catch (error) {
console.error('Error loading jobs:', error);
document.getElementById('jobsContainer').innerHTML =
'<div class="alert alert-danger">Error loading jobs</div>';
}
}
// Load items
async function loadItems() {
try {
const response = await fetch('/api/scrape/items?limit=50');
const items = await response.json();
const tbody = document.getElementById('itemsTableBody');
let html = '';
items.forEach(item => {
const ratingClass = getRatingClass(item.rating_score);
const ratingText = item.rating_score > 0 ? item.rating_score.toFixed(2) : 'Unrated';
html += `
<tr>
<td>
<strong>${item.title || 'No Title'}</strong>
<br><small class="text-muted">${item.domain}</small>
</td>
<td>
<a href="${item.url}" target="_blank" class="text-decoration-none">
${item.url.substring(0, 50)}...
</a>
</td>
<td>
<span class="job-status status-${item.processing_status}">
${item.processing_status}
</span>
</td>
<td>
<span class="rating-badge ${ratingClass}">
${ratingText}
</span>
</td>
<td>
<span class="badge bg-info">${item.language}</span>
</td>
<td>
<span class="badge bg-secondary">${item.word_count}</span>
</td>
<td>
<button class="btn btn-sm btn-outline-primary me-1" onclick="viewItem('${item.id}')">
<i class="fas fa-eye"></i>
</button>
<button class="btn btn-sm btn-outline-success" onclick="rateItem('${item.id}')">
<i class="fas fa-star"></i>
</button>
</td>
</tr>
`;
});
tbody.innerHTML = html;
} catch (error) {
console.error('Error loading items:', error);
document.getElementById('itemsTableBody').innerHTML =
'<tr><td colspan="7" class="text-center text-danger">Error loading items</td></tr>';
}
}
// Load charts
async function loadCharts() {
try {
const [scrapingStats, ratingSummary] = await Promise.all([
fetch('/api/scrape/statistics').then(r => r.json()),
fetch('/api/rating/summary').then(r => r.json())
]);
// Rating distribution chart
const ratingCtx = document.getElementById('ratingChart').getContext('2d');
if (ratingChart) ratingChart.destroy();
ratingChart = new Chart(ratingCtx, {
type: 'doughnut',
data: {
labels: Object.keys(ratingSummary.rating_level_distribution || {}),
datasets: [{
data: Object.values(ratingSummary.rating_level_distribution || {}),
backgroundColor: ['#28a745', '#17a2b8', '#ffc107', '#dc3545', '#6c757d']
}]
},
options: {
responsive: true,
maintainAspectRatio: false,
plugins: {
legend: {
position: 'bottom'
}
}
}
});
// Language distribution chart
const languageCtx = document.getElementById('languageChart').getContext('2d');
if (languageChart) languageChart.destroy();
languageChart = new Chart(languageCtx, {
type: 'bar',
data: {
labels: Object.keys(scrapingStats.language_distribution || {}),
datasets: [{
label: 'Items by Language',
data: Object.values(scrapingStats.language_distribution || {}),
backgroundColor: '#3498db'
}]
},
options: {
responsive: true,
maintainAspectRatio: false,
plugins: {
legend: {
display: false
}
}
}
});
} catch (error) {
console.error('Error loading charts:', error);
}
}
// Form submission
document.getElementById('scrapingForm').addEventListener('submit', async function(e) {
e.preventDefault();
const urls = document.getElementById('urls').value.split('\n').filter(url => url.trim());
const strategy = document.getElementById('strategy').value;
const keywords = document.getElementById('keywords').value.split(',').filter(k => k.trim());
const maxDepth = parseInt(document.getElementById('maxDepth').value);
const delay = parseFloat(document.getElementById('delay').value);
if (urls.length === 0) {
showNotification('Please enter at least one URL', 'warning');
return;
}
const startBtn = document.getElementById('startScrapingBtn');
const originalText = startBtn.innerHTML;
startBtn.innerHTML = '<span class="loading-spinner me-2"></span>Starting...';
startBtn.disabled = true;
try {
const response = await fetch('/api/scrape', {
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
urls: urls,
strategy: strategy,
keywords: keywords.length > 0 ? keywords : null,
max_depth: maxDepth,
delay_between_requests: delay
})
});
const result = await response.json();
if (response.ok) {
showNotification('Scraping job started successfully', 'success');
document.getElementById('scrapingForm').reset();
loadJobs();
} else {
showNotification(result.detail || 'Failed to start scraping job', 'error');
}
} catch (error) {
console.error('Error starting scraping job:', error);
showNotification('Error starting scraping job', 'error');
} finally {
startBtn.innerHTML = originalText;
startBtn.disabled = false;
}
});
// Rate all items
async function rateAllItems() {
try {
const response = await fetch('/api/rating/rate-all', {
method: 'POST'
});
const result = await response.json();
if (response.ok) {
showNotification(`Rated ${result.rated_count} items successfully`, 'success');
loadStatistics();
loadItems();
} else {
showNotification(result.detail || 'Failed to rate items', 'error');
}
} catch (error) {
console.error('Error rating items:', error);
showNotification('Error rating items', 'error');
}
}
// Get low quality items
async function getLowQualityItems() {
try {
const response = await fetch('/api/rating/low-quality');
const result = await response.json();
if (response.ok) {
showNotification(`Found ${result.total_items} low quality items`, 'warning');
// You could display these in a modal or separate section
} else {
showNotification(result.detail || 'Failed to get low quality items', 'error');
}
} catch (error) {
console.error('Error getting low quality items:', error);
showNotification('Error getting low quality items', 'error');
}
}
// Rate specific item
async function rateItem(itemId) {
try {
const response = await fetch(`/api/rating/rate/${itemId}`, {
method: 'POST'
});
const result = await response.json();
if (response.ok) {
showNotification(`Item ${itemId} rated successfully`, 'success');
loadItems();
} else {
showNotification(result.detail || 'Failed to rate item', 'error');
}
} catch (error) {
console.error('Error rating item:', error);
showNotification('Error rating item', 'error');
}
}
// View item details
function viewItem(itemId) {
// This could open a modal with item details
showNotification(`Viewing item ${itemId}`, 'info');
}
// Refresh functions
function refreshStatistics() {
loadStatistics();
showNotification('Statistics refreshed', 'success');
}
function refreshJobs() {
loadJobs();
showNotification('Jobs refreshed', 'success');
}
function refreshItems() {
loadItems();
showNotification('Items refreshed', 'success');
}
// Auto refresh
function startAutoRefresh() {
refreshInterval = setInterval(() => {
loadStatistics();
loadJobs();
}, 10000); // Refresh every 10 seconds
}
// Utility functions
function getRatingClass(score) {
if (score >= 0.8) return 'rating-excellent';
if (score >= 0.6) return 'rating-good';
if (score >= 0.4) return 'rating-average';
if (score >= 0.2) return 'rating-poor';
return 'rating-unrated';
}
function showNotification(message, type = 'info') {
const container = document.getElementById('notificationContainer');
const notification = document.createElement('div');
notification.className = `notification ${type}`;
notification.innerHTML = `
<i class="fas fa-${type === 'success' ? 'check-circle' : type === 'warning' ? 'exclamation-triangle' : type === 'error' ? 'times-circle' : 'info-circle'} me-2"></i>
${message}
`;
container.appendChild(notification);
setTimeout(() => {
notification.classList.add('show');
}, 100);
setTimeout(() => {
notification.classList.remove('show');
setTimeout(() => {
container.removeChild(notification);
}, 300);
}, 5000);
}
// Cleanup on page unload
window.addEventListener('beforeunload', function() {
if (refreshInterval) {
clearInterval(refreshInterval);
}
});
</script>
</body>
</html>