diff --git a/workflow/process_episode.bash b/workflow/process_episode.bash index dbfee95..aebae5b 100755 --- a/workflow/process_episode.bash +++ b/workflow/process_episode.bash @@ -84,13 +84,6 @@ function program_checks() { fi done - if ${force_overwrite} - then - echo_debug "The setting \"force_overwrite\" is set to true, so files will be overwritten." - else - echo_debug "The setting \"force_overwrite\" is set to false, so when files exist the program will skip files if they exist." - fi - } ################################################# @@ -156,23 +149,6 @@ function get_working_dir_from_local_dir() { echo_debug "Processing a local directory" - # Get supplied working dir and ep_num if provided - for argument in "$@" - do - - - if [ -d "${argument}" ] - then - working_dir="$( realpath ${argument} )" - fi - - if [ "$( echo "${argument}" | grep --count "ep_num=" )" -eq "1" ] - then - ep_num="$( echo "${argument}" | sed 's/^.*ep_num=//g' | awk '{print $1}' )" - fi - - done - if [[ ! -d "${working_dir}" || -z "${working_dir}" ]] then echo_error "The working dir is missing. Please supply a working directory." @@ -608,6 +584,7 @@ function generate_initial_report() { +

Skip to Derived Media

Field Mapping

@@ -1098,7 +1075,7 @@ function generate_final_report() { grep -Pv '|' "${working_dir}/processing/${media_basename%.*}_media_report.html" > "${final_report}" - echo "

Text To Speech

+ echo "

Text To Speech

$( echo "${synopsis}" )

@@ -1327,6 +1304,12 @@ function copy_files_to_origin_server() { echo_error "Failed to transfer missing file \"${this_file}\"." ls -al "${this_file}" fi + + if [ "$( ssh rsync.net ls hpr/eps/hpr${ep_num}/ 2>/dev/null | wc -l )" -eq "0" ] + then + echo_debug "Creating \"hpr/eps/hpr${ep_num}/\" on the origin server." + ssh rsync.net mkdir hpr/eps/hpr${ep_num}/ 2>/dev/null + fi rsync --archive --quiet --partial --progress "${this_file}" rsync.net:hpr/eps/hpr${ep_num}/${this_asset} @@ -1347,8 +1330,7 @@ function copy_files_to_origin_server() { } ################################################# -# Send the derived files to the server borg to be sent to the Internet Archive - +# Send the derived files to the server borg to be sent to borg function copy_derived_files_to_borg() { echo_debug "Sending the derived files to the admin server borg" @@ -1426,12 +1408,66 @@ function copy_derived_files_to_borg() { } +################################################# +# Wait for the Internet Archive to finish processing + +function wait_for_the_internet_archive_to_process() { + echo_debug "Sending the derived files to Internet Archive" + + if [ -z "${ep_num}x" ] + then + echo_error "Could not find the episode number while uploading to the Internet Archive" + fi +} + ################################################# # Send the derived files to the server borg to be sent to the Internet Archive function upload_to_the_internet_archive() { echo_debug "Sending the derived files to Internet Archive" + continue + +# hpr4371.flac +# hpr4371.mp3 +# hpr4371.ogg +# hpr4371.opus +# hpr4371_source.flac +# hpr4371.srt +# hpr4371.txt +# hpr4371.wav +# +# for ext in flac mp3 ogg opus wav +# do +# if [[ ! -s "${working_dir}/hpr${ep_num}.${ext}" ]] +# then +# echo_error "The derived files to the Internet Archive are missing \"hpr${ep_num}.${ext}\"." +# ls -al "${working_dir}/hpr${ep_num}.${ext}" +# fi +# done +# +# "${working_dir}/hpr${ep_num}.${ext}" +# +# ep_num="4371" +# ia upload hpr${ep_num} "${working_dir}/hpr${ep_num}.txt" --metadata=mediatype:audio --metadata="contributor:HackerPublicRadio" --metadata="creator:HPR Volunteers" --metadata="date:2025-05-05" --metadata="description:This show is a placeholder and will be updated soon." --metadata="language:eng" --metadata="licenseurl:http://creativecommons.org/licenses/by-sa/4.0" --metadata="title:A placeholder for hpr${ep_num}." --metadata=reviews-allowed:none --header x-archive-keep-old-version:0 --retries=5 --no-derive --no-backup +# +# wait for ia to finish +# ia tasks "hpr${ep_num}" | jq --slurp '[.[] | if .category == "catalog" then .status else empty end] | length' + +# locate placeholder + +# ia upload hpr4321_a686b0995e77c32d9e6566596ed9e4a2 /home/ken/tmp/hpr/processing/1739652162_4321_2025-02-24_624fd1a95db1d05e693eaf06e2f69b8667b0fc42edbc9/hpr4321.txt --metadata=mediatype:audio --metadata="collection:test_collection" --metadata="contributor:HackerPublicRadio" --metadata="creator:Klaatu" --metadata="date:2025-02-24" --metadata="description:This show is a placeholder and will be updated soon." --metadata="language:eng" --metadata="licenseurl:http://creativecommons.org/licenses/by-sa/4.0" --metadata="title:A placeholder for hpr4321." --metadata=reviews-allowed:none --header x-archive-keep-old-version:0 --retries=5 --no-derive --no-backup +# locate placeholder + + +# for extension in .flac .mp3 .ogg .opus _source.ogg .srt .txt .wav; +# do +# echo ia upload hpr${ep_num} "${working_dir}/hpr${ep_num}${extension}" --header "x-archive-keep-old-version:0" --retries=5 --no-derive --no-backup +# done +# +# ia upload hpr hpr4283_image_* --header "x-archive-keep-old-version:0" --retries=5 --no-derive --no-backup + + # # # # for ext in flac mp3 ogg opus wav # # # # do # # # # if [[ ! -s "${working_dir}/hpr${ep_num}.${ext}" ]] @@ -1481,7 +1517,6 @@ function upload_to_the_internet_archive() { # # # # --header x-archive-keep-old-version:0 \ # # # # --retries=5 --no-derive --no-backup # # # # -} # ia upload hpr4295 /data/IA/uploads/hpr4295.wav \ @@ -1501,6 +1536,8 @@ function upload_to_the_internet_archive() { # --header x-archive-keep-old-version:0 \ # --retries=5 --no-derive --no-backup +} + ################################################# # Main exceution starts here # @@ -1517,41 +1554,59 @@ function upload_to_the_internet_archive() { # TODO incorporate direct upload to the IA # TODO copy the files to the backup disk -program_checks # We know that all the programs and variables are set +# Get supplied working dir and ep_num if provided +for argument in "$@" +do -get_working_dir $@ # We have a working directory and a valid json file + if [ "$( echo "${argument}" | grep --count "working_dir=" )" -eq "1" ] + then + working_dir="$( realpath $( echo "${argument}" | sed 's/^.*working_dir=//g' | awk '{print $1}' ) )/" + echo_debug "Using supplied 'working_dir' of \"${working_dir}\"" + fi + + if [ "$( echo "${argument}" | grep --count "ep_num=" )" -eq "1" ] + then + ep_num="$( echo "${argument}" | sed 's/^.*ep_num=//g' | awk '{print $1}' )" + echo_debug "Using supplied 'ep_num' of \"${ep_num}\"" + fi + +done -get_episode_metadata $@ # We have all the metadata we need to process the show. - -extract_images_brute_force # We have extracted the images by brute force - -media_checks # - -generate_initial_report # Generate Initial Report for review by the Janitors - -manual_shownotes_review # Janitors review audio and shownote. Skips if done. - -post_show_to_hpr_db # Posts the episode to HPR. Skips if it is already posted. - -create_tts_summary # Generate text to speech summary - -generate_intro # Generate Intro from the intro theme with overlay of a lead in silence then the tts summary - -generate_parent_audio # Combines the intro, the episode, and the outro to a final cut. - -generate_derived_media # Generate the flac wav mp3 ogg opus files - -generate_show_transcript - -generate_final_report - -manual_final_review - -register_assets +# program_checks # We know that all the programs and variables are set +# +# get_working_dir $@ # We have a working directory and a valid json file +# +# get_episode_metadata $@ # We have all the metadata we need to process the show. +# +# extract_images_brute_force # We have extracted the images by brute force +# +# media_checks # +# +# generate_initial_report # Generate Initial Report for review by the Janitors +# +# manual_shownotes_review # Janitors review audio and shownote. Skips if done. +# +# post_show_to_hpr_db # Posts the episode to HPR. Skips if it is already posted. +# +# create_tts_summary # Generate text to speech summary +# +# generate_intro # Generate Intro from the intro theme with overlay of a lead in silence then the tts summary +# +# generate_parent_audio # Combines the intro, the episode, and the outro to a final cut. +# +# generate_derived_media # Generate the flac wav mp3 ogg opus files +# +# generate_show_transcript +# +# generate_final_report +# +# manual_final_review +# +# register_assets copy_files_to_origin_server -copy_derived_files_to_borg +# copy_derived_files_to_borg # upload_to_the_internet_archive @@ -1559,3 +1614,10 @@ copy_derived_files_to_borg echo_debug "The End" exit 0 + +# if ${force_overwrite} +# then +# echo_debug "The setting \"force_overwrite\" is set to true, so files will be overwritten." +# else +# echo_debug "The setting \"force_overwrite\" is set to false, so when files exist the program will skip files if they exist." +# fi