diff --git a/workflow/process_episode.bash b/workflow/process_episode.bash index 81c6d79..25b0909 100755 --- a/workflow/process_episode.bash +++ b/workflow/process_episode.bash @@ -134,6 +134,27 @@ function check_variable_is_correct() { echo_error "The \"episode_summary_json\" variable has not a valid \"application/json\" mime type." fi ;; + assets_csv) + if [[ ! -s "${assets_csv}" || -z "${assets_csv}" ]] + then + echo_error "The \"assets_csv\" variable/file is missing." + fi + if [ "$( file --brief --mime-type "${assets_csv}" | grep --count 'text/csv' )" -ne "1" ] + then + echo_error "The \"assets_csv\" variable has not a valid \"text/csv\" mime type." + fi + ;; + assets_json) + if [[ ! -s "${assets_json}" || -z "${assets_json}" ]] + then + echo_error "The \"assets_json\" variable/file is missing." + fi + if [ "$( file --brief --mime-type "${assets_json}" | grep --count 'application/json' )" -ne "1" ] + then + echo_error "The \"assets_json\" variable has not a valid \"application/json\" mime type." + fi + ;; + album) if [[ -z "${album}" || "${album}" == "null" ]] then @@ -1410,14 +1431,19 @@ function manual_final_review() { function register_assets() { echo_debug "Registering the assets with the hpr database. register_assets()" + + check_variable_is_correct working_dir ep_num + + assets_csv="${working_dir}/hpr${ep_num}_assets.csv" + assets_json="${working_dir}/hpr${ep_num}_assets.json" - if [[ -s "${working_dir}/hpr${ep_num}_assets.csv" ]] + if [[ -s "${assets_csv}" ]] then - echo_debug "Removing \"${working_dir}/hpr${ep_num}_assets.csv\"." - rm -v "${working_dir}/hpr${ep_num}_assets.csv" + echo_debug "Removing \"${assets_csv}\"." + rm -v "${assets_csv}" fi - echo '"episode_id","filename","extension","size", "sha1sum", "mime_type", "file_type"' | tee "${working_dir}/hpr${ep_num}_assets.csv" + echo '"episode_id","filename","extension","size", "sha1sum", "mime_type", "file_type"' | tee "${assets_csv}" for this_asset_filename in hpr${ep_num}.flac hpr${ep_num}.wav hpr${ep_num}.mp3 hpr${ep_num}.ogg hpr${ep_num}.opus hpr${ep_num}.srt hpr${ep_num}.txt $( find "${working_dir}/" -maxdepth 1 -type f -iname "hpr${ep_num}_image_*.*" ) do @@ -1454,21 +1480,21 @@ function register_assets() { fi done - echo "${ep_num},\"${this_asset_basename}\",\"${this_asset_extension}\",\"${this_asset_size}\",\"${this_asset_sha1sum}\",\"${this_asset_mime_type}\",\"${this_asset_file_type}\"" | tee --append "${working_dir}/hpr${ep_num}_assets.csv" + echo "${ep_num},\"${this_asset_basename}\",\"${this_asset_extension}\",\"${this_asset_size}\",\"${this_asset_sha1sum}\",\"${this_asset_mime_type}\",\"${this_asset_file_type}\"" | tee --append "${assets_csv}" done - if [ -s "${working_dir}/hpr${ep_num}_assets.csv" ] + if [ -s "${assets_csv}" ] then - cat "${working_dir}/hpr${ep_num}_assets.csv" | csvtojson | jq '{"assets":[.[]]}' | tee "${working_dir}/hpr${ep_num}_assets.json" + cat "${assets_csv}" | csvtojson | jq '{"assets":[.[]]}' | tee "assets_json" fi - if [ ! -s "${working_dir}/hpr${ep_num}_assets.json" ] + if [ ! -s "assets_json" ] then - echo_error "The asset json file \"${working_dir}/hpr${ep_num}_assets.json\" is missing."; + echo_error "The asset json file \"assets_json\" is missing."; fi - response="$( curl --silent --netrc-file $HOME/.netrc --write-out '%{http_code}' --output /dev/null --request POST https://hub.hackerpublicradio.org/cms/assets.php --data-ascii @"${working_dir}/hpr${ep_num}_assets.json" --header "Content-Type: application/json" )" + response="$( curl --silent --netrc-file $HOME/.netrc --write-out '%{http_code}' --output /dev/null --request POST https://hub.hackerpublicradio.org/cms/assets.php --data-ascii @"assets_json" --header "Content-Type: application/json" )" if [[ -z "${response}" || "${response}" != "200" ]] then echo_error "The assets for episode hpr${ep_num} has not been registered. The response was \"${response}\"" @@ -1655,125 +1681,59 @@ function create_item_on_the_internet_archive() { # Send the derived files to the server borg to be sent to the Internet Archive function upload_files_to_the_internet_archive() { + echo_debug "Sending the derived files to Internet Archive. upload_files_to_the_internet_archive()" - check_variable_is_correct working_dir ep_num shownotes_html + check_variable_is_correct working_dir ep_num -# hpr4371.flac -# hpr4371.mp3 -# hpr4371.ogg -# hpr4371.opus -# hpr4371_source.flac -# hpr4371.srt -# hpr4371.txt -# hpr4371.wav -# - - if [ "$( find ${working_dir} -mindepth 1 -maxdepth 1 -type f \( -iname "hpr${ep_num}.flac" -or -iname "hpr${ep_num}.mp3" -or -iname "hpr${ep_num}.ogg" -or -iname "hpr${ep_num}.opus" -or -iname "hpr${ep_num}_source.*" -or -iname "hpr${ep_num}.srt" -or -iname "hpr${ep_num}.txt" -or -iname "hpr${ep_num}.wav" \) | wc --lines )" -ne "8" ] + if [ -z "${assets_json}" ] then - echo_error "There are not 8 derived files for the Internet Archive." + assets_json="${working_dir}/episode_summary.json" fi - while read this_ia_file - do - if [[ ! -s "${this_ia_file}" ]] + if [ ! -s "${assets_json}" ] + then + echo_debug "The \"assets_json\" variable/file is missing." + if [ "$( curl --silent --netrc --write-out '%{http_code}' https://hub.hackerpublicradio.org/cms/say.php?id=${ep_num} --output "${assets_json}" )" != 200 ] then - ls -al "${this_ia_file}" - echo_error "The derived files to the Internet Archive are missing \"${this_ia_file}\"." + echo_error "The Episode hpr${ep_num} has not been posted" fi - done < <( find ${working_dir} -mindepth 1 -maxdepth 1 -type f \( -iname "hpr${ep_num}.flac" -or -iname "hpr${ep_num}.mp3" -or -iname "hpr${ep_num}.ogg" -or -iname "hpr${ep_num}.opus" -or -iname "hpr${ep_num}_source.*" -or -iname "hpr${ep_num}.srt" -or -iname "hpr${ep_num}.txt" -or -iname "hpr${ep_num}.wav" \) ) + fi + check_variable_is_correct assets_json -# -# "${working_dir}/hpr${ep_num}.${extension}" -# -# ep_num="4371" -# ia upload hpr${ep_num} "${working_dir}/hpr${ep_num}.txt" --metadata=mediatype:audio --metadata="contributor:HackerPublicRadio" --metadata="creator:HPR Volunteers" --metadata="date:2025-05-05" --metadata="description:This show is a placeholder and will be updated soon." --metadata="language:eng" --metadata="licenseurl:http://creativecommons.org/licenses/by-sa/4.0" --metadata="title:A placeholder for hpr${ep_num}." --metadata=reviews-allowed:none --header x-archive-keep-old-version:0 --retries=5 --no-derive --no-backup -# -# wait for ia to finish -# ia tasks "hpr${ep_num}" | jq --slurp '[.[] | if .category == "catalog" then .status else empty end] | length' -# locate placeholder - -# ia upload hpr4321_a686b0995e77c32d9e6566596ed9e4a2 /home/ken/tmp/hpr/processing/1739652162_4321_2025-02-24_624fd1a95db1d05e693eaf06e2f69b8667b0fc42edbc9/hpr4321.txt --metadata=mediatype:audio --metadata="collection:test_collection" --metadata="contributor:HackerPublicRadio" --metadata="creator:Klaatu" --metadata="date:2025-02-24" --metadata="description:This show is a placeholder and will be updated soon." --metadata="language:eng" --metadata="licenseurl:http://creativecommons.org/licenses/by-sa/4.0" --metadata="title:A placeholder for hpr4321." --metadata=reviews-allowed:none --header x-archive-keep-old-version:0 --retries=5 --no-derive --no-backup -# locate placeholder - - -# for extension in .flac .mp3 .ogg .opus _source.ogg .srt .txt .wav -# do -# echo ia upload hpr${ep_num} "${working_dir}/hpr${ep_num}${extension}" --header "x-archive-keep-old-version:0" --retries=5 --no-derive --no-backup -# done -# -# ia upload hpr hpr4283_image_* --header "x-archive-keep-old-version:0" --retries=5 --no-derive --no-backup - - -# # # # for ext in flac mp3 ogg opus wav -# # # # do -# # # # if [[ ! -s "${working_dir}/hpr${ep_num}.${extension}" ]] -# # # # then -# # # # echo_error "The derived files to the IA are missing \"hpr${ep_num}.${extension}\"." -# # # # ls -al "${working_dir}/hpr${ep_num}.${extension}" -# # # # fi -# # # # done -# # # # -# # # # I don't know if you noticed in the documentation, but there are some key things you need to bear in mind when creating a new IA item: -# # # # -# # # # You MUST upload the metadata first, and the way things seem to be organised, your item MUST have a file with it. That's why my test uploaded that 1 second WAV file. This is a constraint in the ia tool and was in the days of using the S3-like interface, so I imagine it's built-in to the IA software. -# # # # When creating the metadata you MUST define the mediatype, and since this controls some of what you can upload, using audio seems to be vital for HPR stuff. -# # # # I discovered (by accident) that if the metadata phase fails and the next phase (uploading files) continues to run, an item will be created but it will be a default thing which YOU CAN'T CHANGE! You have to plead with the people at info@archive.org to reset this item (they will never delete) because you can't change the defaults to the right values. If this happens and is not fixed, then HPR has effectively lost a slot (at least I tell myself this so that I don't inadvertently screw up an upload). -# # # -# # # ia upload hpr_761c0f557b87090db3f8d4d9bce7fc70 \ -# # # test.wav \ -# # # --metadata="mediatype:audio" \ -# # # --metadata="collection:test_collection" \ -# # # --metadata="collection:podcasts" \ -# # # --metadata="contributor:HPR_test" \ -# # # --metadata="creator:HPR_test" \ -# # # --metadata="date:$(date +%F)" \ -# # # --metadata="description:Summary: Uploading a test item from the command line
Source: We would point to the original show here



This is some test HTML in a test item

" \ -# # # --metadata="language:eng" \ -# # # --metadata="licenseurl:http://creativecommons.org/licenses/by-sa/4.0" \ -# # # --metadata="subject:Test" \ -# # # --metadata="title:hpr_761c0f557b87090db3f8d4d9bce7fc70" \ -# # # --header x-archive-keep-old-version:0 \ -# # # --retries=5 \ -# # # --no-derive \ -# # # --no-backup -# # # # echo ia upload hpr4295 /data/IA/uploads/hpr4295.wav \ -# # # # --metadata="mediatype:audio" \ -# # # # --metadata="collection:hackerpublicradio" \ -# # # # --metadata="collection:podcasts" \ -# # # # --metadata="contributor:HackerPublicRadio" \ -# # # # --metadata="creator:Ken Fallon" \ -# # # # --metadata="date:2025-01-17" \ -# # # # --metadata="description:Summary: Replacing the battery, swapping a fan, and getting a new desktop
Source: https://hackerpublicradio.org/eps/hpr4295/index.html




In this reserve show, Ken

  1. replaces the battery in a SONY A10 telephone .
  2. Swaps out a loud fan for a quiet one in a RIGOL DS1054 .
  3. Then replaces the desktop of an Ikea BEKANT standing desk with a narrower LAGKAPTEN table top not meant for it.
" \ -# # # # --metadata="language:eng" \ -# # # # --metadata="licenseurl:http://creativecommons.org/licenses/by-sa/4.0" \ -# # # # --metadata="subject:IkeaHacks" \ -# # # # --metadata="subject:diy" \ -# # # # --metadata="subject:hardware" \ -# # # # --metadata="title:hpr4295 :: Three Holiday Hacks from 2023" \ -# # # # --header x-archive-keep-old-version:0 \ -# # # # --retries=5 --no-derive --no-backup -# # # # - - -# ia upload hpr4295 /data/IA/uploads/hpr4295.wav \ -# --metadata="mediatype:audio" \ -# --metadata="collection:hackerpublicradio" \ -# --metadata="collection:podcasts" \ -# --metadata="contributor:HackerPublicRadio" \ -# --metadata="creator:Ken Fallon" \ -# --metadata="date:2025-01-17" \ -# --metadata="description:Summary: Replacing the battery, swapping a fan, and getting a new desktop
Source: https://hackerpublicradio.org/eps/hpr4295/index.html




In this reserve show, Ken

  1. replaces the battery in a SONY A10 telephone .
  2. Swaps out a loud fan for a quiet one in a RIGOL DS1054 .
  3. Then replaces the desktop of an Ikea BEKANT standing desk with a narrower LAGKAPTEN table top not meant for it.
" \ -# --metadata="language:eng" \ -# --metadata="licenseurl:http://creativecommons.org/licenses/by-sa/4.0" \ -# --metadata="subject:IkeaHacks" \ -# --metadata="subject:diy" \ -# --metadata="subject:hardware" \ -# --metadata="title:hpr4295 :: Three Holiday Hacks from 2023" \ -# --header x-archive-keep-old-version:0 \ -# --retries=5 --no-derive --no-backup +# if [ "$( find ${working_dir} -mindepth 1 -maxdepth 1 -type f \( -iname "hpr${ep_num}.flac" -or -iname "hpr${ep_num}.mp3" -or -iname "hpr${ep_num}.ogg" -or -iname "hpr${ep_num}.opus" -or -iname "hpr${ep_num}_source.*" -or -iname "hpr${ep_num}.srt" -or -iname "hpr${ep_num}.txt" -or -iname "hpr${ep_num}.wav" \) | wc --lines )" -ne "8" ] +# then +# echo_error "There are not 8 derived files for the Internet Archive." +# fi +# +# while read this_ia_file +# do +# if [[ ! -s "${this_ia_file}" ]] +# then +# ls -al "${this_ia_file}" +# echo_error "The derived files to the Internet Archive are missing \"${this_ia_file}\"." +# fi +# +# echo_debug "Sending \"${this_ia_file}\" to the Internet Archive" +# +# +# # origin_sha1sum="$( echo $( ssh rsync.net "sha1 hpr/eps/hpr${ep_num}/${this_asset}" 2> /dev/null ) | awk '{print $NF}' )" +# # this_asset_sha1sum="$( sha1sum "${this_file}" | awk '{print $1}' )" +# # +# # if [[ -z "${origin_sha1sum}" || -z "${this_asset_sha1sum}" ]] +# # then +# # echo_error "Could not determine the local/origin sha1sum for file \"${this_file}\"." +# # fi +# # +# # ia download hpr_761c0f557b87090db3f8d4d9bce7fc70 hpr_761c0f557b87090db3f8d4d9bce7fc70_files.xml +# +# echo ia upload hpr${ep_num} "${this_ia_file}" --header "x-archive-keep-old-version:0" --retries=5 --no-derive --no-backup +# +# +# +# done < <( find ${working_dir} -mindepth 1 -maxdepth 1 -type f \( -iname "hpr${ep_num}.flac" -or -iname "hpr${ep_num}.mp3" -or -iname "hpr${ep_num}.ogg" -or -iname "hpr${ep_num}.opus" -or -iname "hpr${ep_num}_source.*" -or -iname "hpr${ep_num}.srt" -or -iname "hpr${ep_num}.txt" -or -iname "hpr${ep_num}.wav" \) ) } @@ -1857,9 +1817,9 @@ get_variables_from_episode_summary_json # Get the episode from HPR. # copy_derived_files_to_borg -create_item_on_the_internet_archive +# create_item_on_the_internet_archive -#upload_files_to_the_internet_archive +upload_files_to_the_internet_archive #for i in {4301..4305};do echo ${i};/home/ken/sourcecode/personal/bin/hpr-check-ccdn-links.bash ${i};done