awesome/rust
1
0
Fork 0
mirror of https://github.com/rust-unofficial/awesome-rust.git synced 2024-11-27 11:44:52 -05:00
Code Activity

Add workaround for github actions

Browse source
This commit is contained in:
Tom Parker-Shemilt 2020-01-13 22:13:00 +00:00
parent ff703bac03
commit c25072ec06
2 changed files with 45 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
Cargo.toml
View file

@ -16,3 +16,4 @@ lazy_static = "1"
env_logger = "0.7" env_logger = "0.7"
async-std = "1" async-std = "1"
log = "0.4" log = "0.4"
regex = "1"

74
src/main.rs
View file

@ -1,7 +1,6 @@
use pulldown_cmark::{Parser, Event, Tag}; use pulldown_cmark::{Parser, Event, Tag};
use std::fs; use std::fs;
use futures::future::select_all; use futures::future::{select_all, BoxFuture, FutureExt};
use futures::future::FutureExt;
use std::collections::{BTreeSet, BTreeMap}; use std::collections::{BTreeSet, BTreeMap};
use serde::{Serialize, Deserialize}; use serde::{Serialize, Deserialize};
use anyhow::Result; use anyhow::Result;
@ -12,6 +11,7 @@ use std::time;
use log::{warn, debug}; use log::{warn, debug};
use std::io::Write; use std::io::Write;
use reqwest::{Client, redirect::Policy, StatusCode, header}; use reqwest::{Client, redirect::Policy, StatusCode, header};
use regex::Regex;
struct MaxHandles { struct MaxHandles {
remaining: AtomicU32 remaining: AtomicU32
@ -65,39 +65,51 @@ fn to_anyhow<T, E>(res: std::result::Result<T, E>) -> Result<T>
res.map_err(|x| Into::<anyhow::Error>::into(x)) res.map_err(|x| Into::<anyhow::Error>::into(x))
} }
async fn get_url(url: String) -> (String, Result<String>) { fn get_url(url: String) -> BoxFuture<'static, (String, Result<String>)> {
let _handle = HANDLES.get().await; async move {
let mut res = Err(anyhow::anyhow!("Should always try at least once..")); let _handle = HANDLES.get().await;
for _ in 0..5u8 { let mut res = Err(anyhow::anyhow!("Should always try at least once.."));
debug!("Running {}", url); for _ in 0..5u8 {
let resp = CLIENT debug!("Running {}", url);
.get(&url) let resp = CLIENT
.header(header::ACCEPT, "text/html, */*;q=0.8") .get(&url)
.send() .header(header::ACCEPT, "text/html, */*;q=0.8")
.await; .send()
match resp { .await;
Err(err) => { match resp {
warn!("Error while getting {}, retrying: {}", url, err); Err(err) => {
continue; warn!("Error while getting {}, retrying: {}", url, err);
}
Ok(ref ok) => {
let status = ok.status();
if status != StatusCode::OK {
warn!("Error while getting {}, retrying: {}", url, status);
if status.is_redirection() {
res = Err(anyhow::anyhow!("Got status code {} redirecting to {}", status, ok.headers().get(header::LOCATION).and_then(|h| h.to_str().ok()).unwrap_or("<unknown>")));
} else {
res = Err(anyhow::anyhow!("Got status code {}", status));
}
continue; continue;
} }
Ok(ref ok) => {
let status = ok.status();
if status != StatusCode::OK {
lazy_static! {
static ref ACTIONS_REGEX: Regex = Regex::new(r"https://github.com/(?P<org>[^/]+)/(?P<repo>[^/]+)/actions(?:\?workflow=.+)?").unwrap();
}
if status == StatusCode::NOT_FOUND && ACTIONS_REGEX.is_match(&url) {
let rewritten = ACTIONS_REGEX.replace_all(&url, "https://github.com/$org/$repo");
warn!("Got 404 with Github actions, so replacing {} with {}", url, rewritten);
let (_new_url, res) = get_url(rewritten.to_string()).await;
return (url, res);
}
warn!("Error while getting {}, retrying: {}", url, status);
if status.is_redirection() {
res = Err(anyhow::anyhow!("Got status code {} redirecting to {}", status, ok.headers().get(header::LOCATION).and_then(|h| h.to_str().ok()).unwrap_or("<unknown>")));
} else {
res = Err(anyhow::anyhow!("Got status code {}", status));
}
continue;
}
}
} }
debug!("Finished {}", url);
res = to_anyhow(resp.map(|x| format!("{:?}", x)));
break;
} }
debug!("Finished {}", url); (url, res)
res = to_anyhow(resp.map(|x| format!("{:?}", x))); }.boxed()
break;
}
(url, res)
} }
#[derive(Debug, Serialize, Deserialize)] #[derive(Debug, Serialize, Deserialize)]