diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 8a1f64d..f854c21 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -36,14 +36,26 @@ jobs: name: Find Skin Repositories and Generate Tables run: | set -eo pipefail + mkdir -p "$ARTIFACT_PATH" total_valid_entries=0 + json_valid() { + jq -e . >/dev/null 2>&1 + } + page=1 while :; do users_json=$(curl --retry 3 --retry-delay 5 -sSL \ -H "Authorization: token ${{ secrets.TOKEN }}" \ "$GITEA_API/admin/users?limit=50&page=$page&_ts=$(date +%s)") + + if ! echo "$users_json" | json_valid; then + echo "❌ Invalid users JSON on page $page. Skipping." + echo "$users_json" > "$ARTIFACT_PATH/invalid_users_page_$page.json" + break + fi + count=$(echo "$users_json" | jq 'length') [ "$count" -eq 0 ] && break @@ -53,12 +65,17 @@ jobs: user_login=$(echo "$users_json" | jq -r ".[$i].login") echo "🔍 User: $user_login" - repos_json=$(curl --retry 3 --retry-delay 5 -sSL \ + # Fetch repos + repos_response=$(curl --retry 3 --retry-delay 5 -s -w "%{http_code}" \ -H "Authorization: token ${{ secrets.TOKEN }}" \ "$GITEA_API/users/$user_login/repos?_ts=$(date +%s)") - if ! echo "$repos_json" | jq -e . >/dev/null 2>&1; then - echo " ⚠️ Could not parse repos for $user_login" + repos_code="${repos_response: -3}" + repos_json="${repos_response::-3}" + + if [ "$repos_code" != "200" ] || ! echo "$repos_json" | json_valid; then + echo " ❌ Invalid repo list for $user_login (HTTP $repos_code)" + echo "$repos_json" > "$ARTIFACT_PATH/invalid_repos_${user_login}.json" continue fi @@ -71,34 +88,51 @@ jobs: owner=$(echo "$repos_json" | jq -r ".[$j].owner.login") repo=$(echo "$repos_json" | jq -r ".[$j].name") html_url=$(echo "$repos_json" | jq -r ".[$j].html_url") + echo " → Repo: $owner/$repo" - dir_code=$(curl --retry 3 --retry-delay 5 -s -w "%{http_code}" -o temp_dir.json \ - -H "Authorization: token ${{ secrets.TOKEN }}" \ - "$GITEA_API/repos/$owner/$repo/contents/Skins?_ts=$(date +%s)") + # Check Skins/ directory + dir_response=$(curl --retry 3 --retry-delay 5 -s -w "%{http_code}" \ + -H "Authorization: token ${{ secrets.TOKEN }}" \ + "$GITEA_API/repos/$owner/$repo/contents/Skins?_ts=$(date +%s)") + dir_code="${dir_response: -3}" + if [ "$dir_code" != "200" ]; then echo " ❌ Skipped: No Skins/ directory (HTTP $dir_code)" continue fi - readme_code=$(curl --retry 3 --retry-delay 5 -s -w "%{http_code}" -o temp_readme.json \ - -H "Authorization: token ${{ secrets.TOKEN }}" \ - "$GITEA_API/repos/$owner/$repo/contents/README.md?_ts=$(date +%s)") - if [ "$readme_code" != "200" ]; then - echo " ❌ Skipped: No README.md (HTTP $readme_code)" + # Check README.md + readme_response=$(curl --retry 3 --retry-delay 5 -s -w "%{http_code}" \ + -H "Authorization: token ${{ secrets.TOKEN }}" \ + "$GITEA_API/repos/$owner/$repo/contents/README.md?_ts=$(date +%s)") + readme_code="${readme_response: -3}" + readme_json="${readme_response::-3}" + + if [ "$readme_code" != "200" ] || ! echo "$readme_json" | json_valid; then + echo " ❌ Skipped: No README.md or JSON invalid (HTTP $readme_code)" continue fi - content=$(jq -r .content < temp_readme.json | base64 -d || echo "") - osu_id=$(echo "$content" | awk '/osuid:[ ]*[0-9]+/ { match($0, /[0-9]+/); print substr($0, RSTART, RLENGTH); exit }') + content=$(echo "$readme_json" | jq -r .content | base64 -d || echo "") + + # Extract osuid + osu_id=$(echo "$content" | grep -oE "osuid:[ ]*[0-9]+" | grep -oE "[0-9]+" | head -1) + if [ -z "$osu_id" ]; then echo " ❌ Skipped: No osuid in README" continue fi - user_data=$(curl --retry 3 --retry-delay 5 -s \ + # Fetch osu! user data + user_data=$(curl -s \ "https://osu.ppy.sh/api/get_user?k=${{ secrets.OSUAPIV1 }}&u=$osu_id&type=id&_ts=$(date +%s)") + if ! echo "$user_data" | json_valid; then + echo " ❌ Invalid osu! API response for $osu_id" + continue + fi + if [ "$(echo "$user_data" | jq 'length')" -eq 0 ]; then echo " 🚫 Restricted or banned user" @@ -106,14 +140,11 @@ jobs: pp_rank="RESTRICTED" pp_country_rank="RESTRICTED" padded_rank="9999999" - osu_profile_url="https://osu.ppy.sh/users/$osu_id" - - # ensure timestamp exists before using it - timestamp=$(( $(date +%s) / 86400 * 86400 )) - avatar_url="https://a.ppy.sh/$osu_id?$timestamp" + profile_url="https://osu.ppy.sh/users/$osu_id" + avatar_url="https://a.ppy.sh/$osu_id?$(( $(date +%s)/86400*86400 ))" printf "%s|%s%s%sProfileSkins\n" \ - "$padded_rank" "$username" "$pp_rank" "$pp_country_rank" "$osu_profile_url" "$html_url" >> "$USER_ROWS_FILE" + "$padded_rank" "$username" "$pp_rank" "$pp_country_rank" "$profile_url" "$html_url" >> "$USER_ROWS_FILE" printf "%s|\n" \ "$padded_rank" "$html_url" "$avatar_url" >> "$AVATAR_ROWS_FILE" @@ -122,19 +153,15 @@ jobs: continue fi + # Normal user pp_rank=$(echo "$user_data" | jq -r '.[0].pp_rank // "9999999"') pp_country_rank=$(echo "$user_data" | jq -r '.[0].pp_country_rank // "-"') country=$(echo "$user_data" | jq -r '.[0].country // "-"') username=$(echo "$user_data" | jq -r '.[0].username // "'$owner'"') padded_rank=$(printf "%07d" "$pp_rank") - # Build CC+rank like CH28; fall back sanely if data missing - if [ "$pp_country_rank" != "-" ] && [ -n "$pp_country_rank" ] && [ "$pp_country_rank" != "null" ]; then - if [ "$country" != "-" ] && [ -n "$country" ] && [ "$country" != "null" ]; then - cc_rank="${country}${pp_country_rank}" - else - cc_rank="$pp_country_rank" - fi + if [ "$pp_country_rank" != "-" ] && [ "$country" != "-" ]; then + cc_rank="${country}${pp_country_rank}" else cc_rank="-" fi @@ -143,6 +170,7 @@ jobs: "$padded_rank" "$username" "$pp_rank" "$cc_rank" "$osu_id" "$html_url" >> "$USER_ROWS_FILE" timestamp=$(( $(date +%s) / 86400 * 86400 )) + printf "%s|\n" \ "$padded_rank" "$html_url" "$osu_id" "$timestamp" >> "$AVATAR_ROWS_FILE" @@ -150,6 +178,7 @@ jobs: total_valid_entries=$((total_valid_entries + 1)) done done + page=$((page + 1)) done