From 5196bf42c1eb82372594315da55c87e899d3ac4f Mon Sep 17 00:00:00 2001 From: pommicket Date: Sat, 10 Aug 2024 22:03:27 -0400 Subject: potd --- .gitignore | 1 + game.js | 10 +++- server/Cargo.lock | 10 ++++ server/Cargo.toml | 2 +- server/getfeaturedpictures.py | 20 +++---- server/potd.py | 17 ++++++ server/src/main.rs | 118 +++++++++++++++++++++++++++++++----------- 7 files changed, 137 insertions(+), 41 deletions(-) create mode 100644 .gitignore create mode 100755 server/potd.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..bee8a64 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +__pycache__ diff --git a/game.js b/game.js index d18cb01..484d9c9 100644 --- a/game.js +++ b/game.js @@ -489,6 +489,7 @@ window.addEventListener('load', function () { puzzleHeight = heightFromWidth(puzzleWidth); socket.send(`new ${puzzleWidth} ${puzzleHeight} ${imageUrl}`); } + let waitingForServerToGiveUsImageUrl = false; socket.addEventListener('open', async () => { if (joinPuzzle) { socket.send(`join ${joinPuzzle}`); @@ -496,6 +497,10 @@ window.addEventListener('load', function () { hostPuzzle(); } else if (imageUrl === 'randomFeaturedWikimedia') { socket.send('randomFeaturedWikimedia'); + waitingForServerToGiveUsImageUrl = true; + } else if (imageUrl === 'wikimediaPotd') { + socket.send('wikimediaPotd'); + waitingForServerToGiveUsImageUrl = true; } else { // TODO : better error reporting throw new Error("bad image URL"); @@ -511,8 +516,9 @@ window.addEventListener('load', function () { piece.upToDateWithServer = true; } receivedAck = true; - } else if (e.data.startsWith('wikimediaImage ')) { - imageUrl = decodeURI(e.data.substring('wikimediaImage '.length)); + } else if (waitingForServerToGiveUsImageUrl && e.data.startsWith('useImage ')) { + waitingForServerToGiveUsImageUrl = false; + imageUrl = decodeURI(e.data.substring('useImage '.length)); hostPuzzle(); } } else { diff --git a/server/Cargo.lock b/server/Cargo.lock index 6e02d34..c7e5745 100644 --- a/server/Cargo.lock +++ b/server/Cargo.lock @@ -473,6 +473,15 @@ dependencies = [ "digest", ] +[[package]] +name = "signal-hook-registry" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" +dependencies = [ + "libc", +] + [[package]] name = "slab" version = "0.4.9" @@ -556,6 +565,7 @@ dependencies = [ "libc", "mio", "pin-project-lite", + "signal-hook-registry", "socket2", "tokio-macros", "windows-sys", diff --git a/server/Cargo.toml b/server/Cargo.toml index 8ef7d7b..49d409e 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -8,6 +8,6 @@ anyhow = "1.0.86" futures-util = "0.3" rand = { version = "0.8.5", features = ["std", "std_rng"] } sled = "0.34.7" -tokio = { version = "1", features = ["rt", "macros", "rt-multi-thread", "net", "io-util", "sync", "time"] } +tokio = { version = "1", features = ["rt", "macros", "rt-multi-thread", "net", "io-util", "sync", "time", "process"] } tokio-tungstenite = "0.23.1" tungstenite = "0.23.0" diff --git a/server/getfeaturedpictures.py b/server/getfeaturedpictures.py index 916c026..c15953f 100755 --- a/server/getfeaturedpictures.py +++ b/server/getfeaturedpictures.py @@ -16,7 +16,7 @@ def make_file_request(cmcontinue): break return json.loads(response.text) -def make_url_request(images): +def get_urls_of_images(images): while True: time.sleep(1) url = 'https://commons.wikimedia.org/w/api.php?action=query&format=json&maxlag=5&prop=imageinfo&iiprop=url&titles=' + urllib.parse.quote('|'.join(images)) @@ -24,9 +24,10 @@ def make_url_request(images): if 'X-Database-Lag' in response.headers: time.sleep(5) break - return json.loads(response.text) - -def get_files(): + response = json.loads(response.text) + return [page['imageinfo'][0]['url'] for page in response['query']['pages'].values()] + +def get_featured_files(): with open('featuredpictures_files.txt', 'w') as f: cmcontinue = '' count = 0 @@ -47,14 +48,15 @@ def get_files(): print('no continue! done probably') break -def get_urls(): +def get_featured_urls(): with open('featuredpictures_files.txt', 'r') as f: files = [line.strip() for line in f] with open('featuredpictures.txt', 'w') as f: for i in range(0, len(files), 30): print('got URLs for',i,'files') batch = files[i:min(len(files), i + 30)] - response = make_url_request(batch) - f.write(''.join(page['imageinfo'][0]['url'] + '\n' for page in response['query']['pages'].values())) -get_files() -get_urls() + urls = get_urls(batch) + f.write(''.join(url + '\n' for url in urls)) +if __name__ == '__main__': + get_featured_files() + get_featured_urls() diff --git a/server/potd.py b/server/potd.py new file mode 100755 index 0000000..61c1f88 --- /dev/null +++ b/server/potd.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 +import requests +from xml.etree import ElementTree +from getfeaturedpictures import get_urls_of_images +headers = {'Accept-Encoding':'gzip', 'User-Agent': 'contact pommicket+jigsaw @ gmail.com '} + +URL = 'https://commons.wikimedia.org/w/api.php?action=featuredfeed&feed=potd&feedformat=rss&maxlag=5' + +response = requests.get(URL, headers=headers).text +xml = ElementTree.fromstring(response) +item = xml.findall('channel/item')[-1] +desc = item.find('description').text +start = desc.index('"/wiki/File:') + len('"/wiki/') +end = desc.index('"', start) +name = desc[start:end] +url = get_urls_of_images([name])[0] +print(url) diff --git a/server/src/main.rs b/server/src/main.rs index 442b6bf..c289c89 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -1,13 +1,13 @@ use anyhow::anyhow; use futures_util::{SinkExt, StreamExt}; -use rand::Rng; use rand::seq::SliceRandom; +use rand::Rng; use std::io::prelude::*; use std::net::SocketAddr; -use std::sync::LazyLock; +use std::time::{Duration, SystemTime}; use tokio::io::AsyncWriteExt; +use tokio::sync::RwLock; use tungstenite::protocol::Message; -use std::time::{SystemTime, Duration}; const PUZZLE_ID_CHARSET: &[u8] = b"23456789abcdefghijkmnopqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ"; const PUZZLE_ID_LEN: usize = 7; @@ -17,11 +17,13 @@ fn generate_puzzle_id() -> [u8; PUZZLE_ID_LEN] { [(); 7].map(|()| *PUZZLE_ID_CHARSET.choose(&mut rng).unwrap()) } +#[derive(Debug)] struct Server { puzzles: sled::Tree, pieces: sled::Tree, connectivity: sled::Tree, wikimedia_featured: Vec, + wikimedia_potd: RwLock, } fn get_puzzle_info(server: &Server, id: &[u8]) -> anyhow::Result> { @@ -84,7 +86,10 @@ async fn handle_connection( return Err(anyhow!("too many pieces")); } let mut puzzle_data = vec![width, height]; - let timestamp: u64 = SystemTime::now().duration_since(SystemTime::UNIX_EPOCH).expect("time went backwards :/").as_secs(); + let timestamp: u64 = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .expect("time went backwards :/") + .as_secs(); for byte in timestamp.to_le_bytes() { puzzle_data.push(byte); } @@ -126,35 +131,37 @@ async fn handle_connection( for x in 0..(width as u16) { let dx: f32 = rng.gen_range(0.0..0.5); let dy: f32 = rng.gen_range(0.0..0.5); - positions.push([(x as f32 + dx) / ((width + 1) as f32), (y as f32 + dy) / ((height + 1) as f32)]); + positions.push([ + (x as f32 + dx) / ((width + 1) as f32), + (y as f32 + dy) / ((height + 1) as f32), + ]); } } positions.shuffle(&mut rng); // rust isn't smart enough to do the zero-copy with f32::to_le_bytes and Vec::into_flattened let ptr: *mut [[f32; 2]] = Box::into_raw(positions.into_boxed_slice()); - let ptr: *mut [u8] = std::ptr::slice_from_raw_parts_mut(ptr.cast(), (width as usize) * (height as usize) * 8); + let ptr: *mut [u8] = std::ptr::slice_from_raw_parts_mut( + ptr.cast(), + (width as usize) * (height as usize) * 8, + ); // evil unsafe code >:3 pieces_data = unsafe { Box::from_raw(ptr) }; - } server.pieces.insert(id, pieces_data)?; - let mut connectivity_data = Vec::new(); - connectivity_data.resize((width as usize) * (height as usize) * 2, 0); - let mut it = connectivity_data.iter_mut(); + let mut connectivity_data = + Vec::with_capacity((width as usize) * (height as usize) * 2); for i in 0..(width as u16) * (height as u16) { - let [a, b] = i.to_le_bytes(); - *it.next().unwrap() = a; - *it.next().unwrap() = b; + connectivity_data.extend(i.to_le_bytes()); } server.connectivity.insert(id, connectivity_data)?; ws.send(Message::Text(format!("id: {}", std::str::from_utf8(&id)?))) .await?; - let info = get_puzzle_info(&server, &id)?; + let info = get_puzzle_info(server, &id)?; ws.send(Message::Binary(info)).await?; } else if let Some(id) = text.strip_prefix("join ") { let id = id.as_bytes().try_into()?; puzzle_id = Some(id); - let info = get_puzzle_info(&server, &id)?; + let info = get_puzzle_info(server, &id)?; ws.send(Message::Binary(info)).await?; } else if text.starts_with("move ") { let puzzle_id = puzzle_id.ok_or_else(|| anyhow!("move without puzzle ID"))?; @@ -177,7 +184,7 @@ async fn handle_connection( loop { let curr_pieces = server .pieces - .get(&puzzle_id)? + .get(puzzle_id)? .ok_or_else(|| anyhow!("bad puzzle ID"))?; let mut new_pieces = curr_pieces.to_vec(); for Motion { piece, x, y } in motions.iter().copied() { @@ -192,7 +199,7 @@ async fn handle_connection( } if server .pieces - .compare_and_swap(&puzzle_id, Some(curr_pieces), Some(new_pieces))? + .compare_and_swap(puzzle_id, Some(curr_pieces), Some(new_pieces))? .is_ok() { break; @@ -208,7 +215,7 @@ async fn handle_connection( loop { let curr_connectivity = server .connectivity - .get(&puzzle_id)? + .get(puzzle_id)? .ok_or_else(|| anyhow!("bad puzzle ID"))?; let mut new_connectivity = curr_connectivity.to_vec(); if piece1 >= curr_connectivity.len() / 2 @@ -235,7 +242,7 @@ async fn handle_connection( if server .connectivity .compare_and_swap( - &puzzle_id, + puzzle_id, Some(curr_connectivity), Some(new_connectivity), )? @@ -249,11 +256,11 @@ async fn handle_connection( let puzzle_id = puzzle_id.ok_or_else(|| anyhow!("poll without puzzle ID"))?; let pieces = server .pieces - .get(&puzzle_id)? + .get(puzzle_id)? .ok_or_else(|| anyhow!("bad puzzle ID"))?; let connectivity = server .connectivity - .get(&puzzle_id)? + .get(puzzle_id)? .ok_or_else(|| anyhow!("bad puzzle ID"))?; let mut data = vec![2, 0, 0, 0, 0, 0, 0, 0]; // opcode / version number + padding data.extend_from_slice(&pieces); @@ -262,10 +269,16 @@ async fn handle_connection( } else if text == "randomFeaturedWikimedia" { let choice = rand::thread_rng().gen_range(0..server.wikimedia_featured.len()); ws.send(Message::Text(format!( - "wikimediaImage {}", + "useImage {}", server.wikimedia_featured[choice] ))) .await?; + } else if text == "wikimediaPotd" { + ws.send(Message::Text(format!( + "useImage {}", + server.wikimedia_potd.read().await + ))) + .await?; } } } @@ -278,6 +291,23 @@ fn read_to_lines(path: &str) -> std::io::Result> { reader.lines().collect() } +async fn try_get_potd() -> anyhow::Result { + let output = tokio::process::Command::new("python3") + .arg("potd.py") + .output() + .await?; + Ok(String::from_utf8(output.stdout)?.trim().to_string()) +} +async fn get_potd() -> String { + match try_get_potd().await { + Ok(s) => s, + Err(e) => { + eprintln!("couldn't get potd: {e}"); + String::new() + } + } +} + #[tokio::main] async fn main() { let port = 54472; @@ -289,7 +319,9 @@ async fn main() { return; } }; - static SERVER_VALUE: LazyLock = LazyLock::new(|| { + let start_time = SystemTime::now(); + // leak this since we need all threads to be able to access this + let server: &'static Server = Box::leak(Box::new({ let wikimedia_featured = read_to_lines("featuredpictures.txt").expect("Couldn't read featuredpictures.txt"); let db = sled::open("database.sled").expect("error opening database"); @@ -298,14 +330,30 @@ async fn main() { let connectivity = db .open_tree("CONNECTIVITY") .expect("error opening connectivity tree"); + let potd = get_potd().await; Server { puzzles, pieces, connectivity, + wikimedia_potd: RwLock::new(potd), wikimedia_featured, } + })); + tokio::task::spawn(async move { + fn next_day(t: SystemTime) -> SystemTime { + let day = 60 * 60 * 24; + let dt = t.duration_since(SystemTime::UNIX_EPOCH).unwrap().as_secs(); + SystemTime::UNIX_EPOCH + Duration::from_secs((dt + day - 1) / day * day) + } + let mut last_time = start_time; + loop { + let time_to_sleep = next_day(last_time).duration_since(last_time).unwrap(); + tokio::time::sleep(time_to_sleep).await; + let potd = get_potd().await; + *server.wikimedia_potd.write().await = potd; + last_time = SystemTime::now(); + } }); - let server: &Server = &SERVER_VALUE; tokio::task::spawn(async { loop { // TODO : sweep @@ -314,17 +362,29 @@ async fn main() { for item in server.puzzles.iter() { let (key, value) = item.expect("sweep failed to read database"); let timestamp: [u8; 8] = value[2..2 + 8].try_into().unwrap(); - let timestamp = SystemTime::UNIX_EPOCH + Duration::from_secs(u64::from_le_bytes(timestamp)); - if now.duration_since(timestamp).unwrap_or_default() >= Duration::from_secs(60 * 60 * 24 * 7) { + let timestamp = + SystemTime::UNIX_EPOCH + Duration::from_secs(u64::from_le_bytes(timestamp)); + if now.duration_since(timestamp).unwrap_or_default() + >= Duration::from_secs(60 * 60 * 24 * 7) + { // delete puzzles created at least 1 week ago to_delete.push(key); } } for key in to_delete { // technically there is a race condition here but stop being silly - server.puzzles.remove(&key).expect("sweep failed to delete entry"); - server.pieces.remove(&key).expect("sweep failed to delete entry"); - server.connectivity.remove(&key).expect("sweep failed to delete entry"); + server + .puzzles + .remove(&key) + .expect("sweep failed to delete entry"); + server + .pieces + .remove(&key) + .expect("sweep failed to delete entry"); + server + .connectivity + .remove(&key) + .expect("sweep failed to delete entry"); } tokio::time::sleep(std::time::Duration::from_secs(3600)).await; } -- cgit v1.2.3