#!/usr/bin/env python3 """ TikTok Worker - Queue Mode Stalno provjerava ima li posla i odmah ga izvršava Pokreni: python app.py Dashboard: http://localhost:5000 """ import os import sys import json import time import threading from datetime import datetime from flask import Flask, render_template_string, jsonify, request from flask_socketio import SocketIO # ============================================ # KONFIGURACIJA # ============================================ CONFIG_FILE = 'config.json' DEFAULT_CONFIG = { 'server_url': 'http://localhost/ntts/ntts2/dedicated', 'api_key': '', 'videos_new': 30, 'videos_existing': 10, 'poll_interval': 5, # Koliko cesto provjerava ima li posla (sekunde) } def load_config(): if os.path.exists(CONFIG_FILE): with open(CONFIG_FILE, 'r') as f: return {**DEFAULT_CONFIG, **json.load(f)} return DEFAULT_CONFIG.copy() def save_config(cfg): with open(CONFIG_FILE, 'w') as f: json.dump(cfg, f, indent=2) config = load_config() # ============================================ # FLASK APP # ============================================ app = Flask(__name__) app.config['SECRET_KEY'] = 'tiktok-worker-secret' socketio = SocketIO(app, cors_allowed_origins="*", async_mode='threading') # Status worker_status = { 'active': False, # Worker NIJE aktivan dok korisnik ne klikne Start 'working': False, # Da li trenutno radi scrape 'waiting': False, # Da li ceka na posao 'current_channel': None, 'last_check': None, 'total_synced': 0, 'total_failed': 0, 'queue_size': 0 } worker_thread = None worker_should_stop = False log_buffer = [] def log(message, level='info'): timestamp = datetime.now().strftime('%H:%M:%S') entry = {'time': timestamp, 'message': str(message), 'level': level} log_buffer.append(entry) if len(log_buffer) > 500: log_buffer.pop(0) try: print(f"[{timestamp}] {message}") except: pass socketio.emit('log', entry) def broadcast_status(): socketio.emit('status', worker_status) # ============================================ # YT-DLP SCRAPER # ============================================ def scrape_with_ytdlp(username, video_count): import yt_dlp url = f"https://www.tiktok.com/@{username}" user_info = {'username': username, 'nickname': username} # Trazimo vise videa da bi pokrili eventualne greske/pinned videe search_limit = video_count + 30 for attempt in range(3): videos = [] # Provjeri cookies.txt za precizniji scrape (vidi SVE videe) cookie_file = 'cookies.txt' if os.path.exists('cookies.txt') else None ydl_opts = { 'quiet': True, 'no_warnings': True, 'extract_flat': True, # BITNO: True samo kupi podatke sa liste (brze i manje blokiranja) 'ignoreerrors': True, 'playlistend': search_limit, 'skip_download': True, 'no_check_certificate': True, 'socket_timeout': 30, 'retries': 5, 'http_headers': { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36', 'Accept-Language': 'en-US,en;q=0.9', 'Referer': 'https://www.tiktok.com/', } } if cookie_file: ydl_opts['cookiefile'] = cookie_file if attempt == 0: log(" [Auth] Koristim cookies.txt + Flat extraction", 'success') try: with yt_dlp.YoutubeDL(ydl_opts) as ydl: result = ydl.extract_info(url, download=False) if not result or not result.get('entries'): if attempt < 2: time.sleep(3) continue return {'success': False, 'user': user_info, 'videos': []} user_info = { 'username': username, 'nickname': result.get('uploader', username), 'avatar': result.get('thumbnail', ''), 'followers': result.get('channel_follower_count', 0), 'video_count': result.get('playlist_count', 0) } entries = result.get('entries', []) valid_entries = 0 for v in entries: if len(videos) >= video_count: break if not v: continue try: desc = v.get('description', '') or '' title = v.get('title', '') or desc[:200] or 'No Title' # Parsiraj datum posted_at = v.get('timestamp', 0) if not posted_at and v.get('upload_date'): try: posted_at = datetime.strptime(v.get('upload_date'), '%Y%m%d').timestamp() except: posted_at = 0 videos.append({ 'tiktok_id': str(v.get('id', '')), 'title': title, 'views': v.get('view_count', 0) or 0, 'likes': v.get('like_count', 0) or 0, 'comments': v.get('comment_count', 0) or 0, 'shares': v.get('repost_count', 0) or 0, 'posted_at': posted_at or 0, 'url': v.get('webpage_url', ''), 'thumbnail': v.get('thumbnail', ''), 'duration': v.get('duration', 0) or 0 }) valid_entries += 1 except: continue if videos: # Log datum najnovijeg videa za provjeru try: latest = max(videos, key=lambda x: x['posted_at']) date_str = datetime.fromtimestamp(latest['posted_at']).strftime('%d.%m.%Y %H:%M') if latest['posted_at'] > 0 else "NEMA DATUMA" log(f" Najnoviji video: {date_str}", 'success') except: pass return {'success': True, 'user': user_info, 'videos': videos} except Exception as e: if attempt < 2: log(f" Retry {attempt+2}/3 zbog greske: {str(e)[:50]}...", 'warning') time.sleep(3) continue return {'success': False, 'user': user_info, 'videos': []} # ============================================ # API CALLS # ============================================ def api_call(endpoint, method='GET', data=None): import requests url = f"{config['server_url']}/api/remote-sync.php?action={endpoint}" headers = {'X-API-Key': config['api_key'], 'Content-Type': 'application/json'} try: log(f" [API] {method} {endpoint}...", 'info') if method == 'GET': start = time.time() r = requests.get(url, headers=headers, timeout=30) elapsed = time.time() - start log(f" [API] GET odgovor za {elapsed:.1f}s", 'success') else: # Detaljno mjerenje vremena za POST data_json = json.dumps(data) if data else '{}' data_size = len(data_json) log(f" [API] Saljem {data_size} bytes...", 'info') # Koristimo session za bolji timing session = requests.Session() # Mjerimo vrijeme konekcije t1 = time.time() # Kreiraj request objekt req = requests.Request('POST', url, headers=headers, json=data) prepared = session.prepare_request(req) t2 = time.time() log(f" [API] Request pripremljen za {(t2-t1)*1000:.0f}ms", 'info') # Pošalji i čekaj odgovor r = session.send(prepared, timeout=(10, 120)) # (connect_timeout, read_timeout) t3 = time.time() total = t3 - t1 send_time = t3 - t2 log(f" [API] Server odgovorio za {send_time:.1f}s (ukupno: {total:.1f}s, status: {r.status_code})", 'success' if r.status_code == 200 else 'warning') # Ispisi odgovor ako ima timing info try: resp_data = r.json() if 'timings' in resp_data: log(f" [API] Server timings: {resp_data['timings']}", 'info') return resp_data except: return r.json() return r.json() except requests.exceptions.ConnectTimeout: log(f" [API] CONNECT TIMEOUT - server nedostupan!", 'error') return {'success': False, 'error': 'Connect timeout'} except requests.exceptions.ReadTimeout: log(f" [API] READ TIMEOUT - server ne odgovara!", 'error') return {'success': False, 'error': 'Read timeout'} except Exception as e: log(f" [API] ERROR: {str(e)[:80]}", 'error') return {'success': False, 'error': str(e)} # ============================================ # WORKER QUEUE LOOP # ============================================ def worker_queue_loop(): """Kontinuirano provjerava ima li posla""" global worker_status, config, worker_should_stop log("=" * 40) log("WORKER POKRENUT - Queue Mode") log(f"Polling: svake {config.get('poll_interval', 5)}s") log("=" * 40) worker_should_stop = False while not worker_should_stop: try: worker_status['last_check'] = datetime.now().strftime('%H:%M:%S') worker_status['waiting'] = True worker_status['working'] = False broadcast_status() # Provjeri ima li posla jobs = api_call('pending-jobs') if not jobs.get('success'): log(f"API error: {jobs.get('error')}", 'error') time.sleep(10) continue channels = jobs.get('jobs', []) worker_status['queue_size'] = len(channels) if not channels: # Nema posla - cekaj pa provjeri opet time.sleep(config.get('poll_interval', 5)) continue # IMA POSLA - odmah radi! worker_status['waiting'] = False worker_status['working'] = True broadcast_status() log(f"Novi posao: {len(channels)} kanala") for i, ch in enumerate(channels, 1): username = ch['username'] is_new = int(ch.get('is_new', 0)) == 1 existing = int(ch.get('existing_videos', 0)) video_count = config['videos_new'] if (is_new or existing == 0) else config['videos_existing'] # Provjeri stop PRIJE obrade svakog kanala if worker_should_stop: log("Stop signal primljen, prekidam...", 'warning') break worker_status['current_channel'] = f"@{username}" broadcast_status() log(f"[{i}/{len(channels)}] @{username}") # Scrape result = scrape_with_ytdlp(username, video_count) # Provjeri stop NAKON scrape-a if worker_should_stop: log("Stop signal primljen, prekidam...", 'warning') break # Ako je scrape potpuno neuspjesan (nema user info), onda je fail # Ali ako nema videa, a ima user info, to je validan prazan kanal user_ok = result.get('user', {}).get('nickname') if not user_ok and not result.get('success'): log(f" FAIL - nemoguce dohvatiti kanal", 'error') worker_status['total_failed'] += 1 continue if not result.get('videos'): log(f" Info: Kanal nema videa ili su nedostupni", 'warning') else: log(f" Dohvaceno {len(result['videos'])} videa") # Provjeri stop PRIJE sync-a if worker_should_stop: log("Stop signal primljen, prekidam...", 'warning') break # Sync sync = api_call('sync-channel', 'POST', { 'channel_id': ch['id'], 'username': username, 'user': result.get('user'), 'videos': result.get('videos') }) if sync.get('success'): new_v = sync.get('new_videos', 0) upd_v = sync.get('updated_videos', 0) log(f" OK: +{new_v} new, ~{upd_v} updated", 'success') worker_status['total_synced'] += 1 else: log(f" SYNC ERROR: {sync.get('error')}", 'error') worker_status['total_failed'] += 1 broadcast_status() time.sleep(1) # Mala pauza izmedju kanala worker_status['current_channel'] = None worker_status['queue_size'] = 0 if not worker_should_stop: log("Posao zavrsen, cekam novi...") except Exception as e: log(f"Error: {e}", 'error') time.sleep(5) # Worker zaustavljen worker_status['active'] = False worker_status['working'] = False worker_status['waiting'] = False broadcast_status() log("WORKER ZAUSTAVLJEN", 'warning') # ============================================ # DASHBOARD HTML # ============================================ DASHBOARD_HTML = ''' TikTok Worker
0
Synced
0
Failed
0
Queue
-
Last Check
Worker automatski procesira sve zahtjeve sa servera
Live Console
Config
Uploaduj 'cookies.txt' za pristup 18+/pinned videima.
Koliko cesto provjerava server
''' @app.route('/') def dashboard(): return render_template_string(DASHBOARD_HTML, config=config) @app.route('/api/status') def get_status(): return jsonify(worker_status) @app.route('/api/logs') def get_logs(): return jsonify(log_buffer[-200:]) @app.route('/api/config', methods=['GET', 'POST']) def handle_config(): global config if request.method == 'POST': data = request.get_json() config.update(data) save_config(config) return jsonify({'success': True}) return jsonify(config) @app.route('/api/upload-cookies', methods=['POST']) def upload_cookies(): try: if 'file' not in request.files: return jsonify({'success': False, 'error': 'Nema fajla'}) file = request.files['file'] if file.filename == '': return jsonify({'success': False, 'error': 'Nije odabran fajl'}) # Sacuvaj kao cookies.txt file.save('cookies.txt') log("Novi cookies.txt USPJESNO uploadovan!", 'success') # Update status odma worker_status['cookies_present'] = True broadcast_status() return jsonify({'success': True, 'message': 'Cookies sacuvani!'}) except Exception as e: log(f"Upload greska: {e}", 'error') return jsonify({'success': False, 'error': str(e)}) @app.route('/api/delete-cookies', methods=['POST']) def delete_cookies(): try: if os.path.exists('cookies.txt'): os.remove('cookies.txt') worker_status['cookies_present'] = False broadcast_status() log("cookies.txt obrisan", 'warning') return jsonify({'success': True}) except Exception as e: return jsonify({'success': False, 'error': str(e)}) @app.route('/api/start', methods=['POST']) def start_worker(): global worker_thread, worker_should_stop if worker_status['active']: return jsonify({'success': False, 'error': 'Worker vec radi!'}) worker_should_stop = False worker_status['active'] = True worker_thread = threading.Thread(target=worker_queue_loop, daemon=True) worker_thread.start() broadcast_status() return jsonify({'success': True, 'message': 'Worker pokrenut!'}) @app.route('/api/stop', methods=['POST']) def stop_worker(): global worker_should_stop if not worker_status['active']: return jsonify({'success': False, 'error': 'Worker vec ne radi!'}) worker_should_stop = True log("Zaustavljam worker...", 'warning') return jsonify({'success': True, 'message': 'Worker se zaustavlja...'}) # ============================================ # DIRECT SCRAPE API (za PHP server) # ============================================ @app.route('/api/scrape', methods=['POST']) def scrape_direct(): """ Direktan scrape jednog kanala - poziva ga PHP server. Prima: {username: string, channel_id?: int} Vraća: {success, user, videos} """ try: data = request.get_json() or {} username = data.get('username', '').strip().lstrip('@') channel_id = data.get('channel_id') if not username: return jsonify({'success': False, 'error': 'Username obavezan'}) log(f"[SCRAPE] Direktan zahtjev za @{username}", 'info') # Odredi broj videa za dohvat video_count = config.get('videos_existing', 20) # Scrape odmah result = scrape_with_ytdlp(username, video_count) if result['success']: log(f"[SCRAPE] @{username}: {len(result.get('videos', []))} videa", 'success') else: log(f"[SCRAPE] @{username}: Greška - {result.get('error', 'nepoznato')}", 'error') # Dodaj channel_id u odgovor ako je poslan if channel_id: result['channel_id'] = channel_id return jsonify(result) except Exception as e: log(f"[SCRAPE] Greška: {str(e)}", 'error') return jsonify({'success': False, 'error': str(e)}) @app.route('/api/scrape-batch', methods=['POST']) def scrape_batch(): """ Scrape više kanala odjednom. Prima: {channels: [{username, channel_id}, ...]} Vraća: {success, results: [...]} """ try: data = request.get_json() or {} channels = data.get('channels', []) if not channels: return jsonify({'success': False, 'error': 'Nema kanala za scrape'}) log(f"[BATCH] Scrape {len(channels)} kanala", 'info') results = [] video_count = config.get('videos_existing', 20) for ch in channels: username = ch.get('username', '').strip().lstrip('@') channel_id = ch.get('channel_id') if not username: continue log(f"[BATCH] Scraping @{username}...", 'info') result = scrape_with_ytdlp(username, video_count) result['channel_id'] = channel_id result['username'] = username results.append(result) if result['success']: log(f"[BATCH] @{username}: OK ({len(result.get('videos', []))} videa)", 'success') else: log(f"[BATCH] @{username}: Greška", 'warning') log(f"[BATCH] Završeno - {len(results)} kanala", 'success') return jsonify({ 'success': True, 'count': len(results), 'results': results }) except Exception as e: log(f"[BATCH] Greška: {str(e)}", 'error') return jsonify({'success': False, 'error': str(e)}) # ============================================ # MAIN # ============================================ if __name__ == '__main__': # Initial verification if os.path.exists('cookies.txt'): worker_status['cookies_present'] = True print("[INFO] cookies.txt pronadjen - Auth mode aktivan") else: worker_status['cookies_present'] = False print("[INFO] cookies.txt nije pronadjen - Guest mode") print("=" * 40) print("TikTok Worker - Queue Mode") print("=" * 40) print(f"Server: {config['server_url']}") print("Dashboard: http://0.0.0.0:5000") print("Worker ceka na START komandu iz Web UI-a") print("=" * 40) # NE POKRECI AUTOMATSKI - cekaj na korisnika socketio.run(app, host='0.0.0.0', port=5000, debug=False, allow_unsafe_werkzeug=True)