more changes
This commit is contained in:
parent
35305a5c45
commit
d7bee0be56
@ -134,6 +134,27 @@ function check_variable_is_correct() {
|
|||||||
echo_error "The \"episode_summary_json\" variable has not a valid \"application/json\" mime type."
|
echo_error "The \"episode_summary_json\" variable has not a valid \"application/json\" mime type."
|
||||||
fi
|
fi
|
||||||
;;
|
;;
|
||||||
|
assets_csv)
|
||||||
|
if [[ ! -s "${assets_csv}" || -z "${assets_csv}" ]]
|
||||||
|
then
|
||||||
|
echo_error "The \"assets_csv\" variable/file is missing."
|
||||||
|
fi
|
||||||
|
if [ "$( file --brief --mime-type "${assets_csv}" | grep --count 'text/csv' )" -ne "1" ]
|
||||||
|
then
|
||||||
|
echo_error "The \"assets_csv\" variable has not a valid \"text/csv\" mime type."
|
||||||
|
fi
|
||||||
|
;;
|
||||||
|
assets_json)
|
||||||
|
if [[ ! -s "${assets_json}" || -z "${assets_json}" ]]
|
||||||
|
then
|
||||||
|
echo_error "The \"assets_json\" variable/file is missing."
|
||||||
|
fi
|
||||||
|
if [ "$( file --brief --mime-type "${assets_json}" | grep --count 'application/json' )" -ne "1" ]
|
||||||
|
then
|
||||||
|
echo_error "The \"assets_json\" variable has not a valid \"application/json\" mime type."
|
||||||
|
fi
|
||||||
|
;;
|
||||||
|
|
||||||
album)
|
album)
|
||||||
if [[ -z "${album}" || "${album}" == "null" ]]
|
if [[ -z "${album}" || "${album}" == "null" ]]
|
||||||
then
|
then
|
||||||
@ -1410,14 +1431,19 @@ function manual_final_review() {
|
|||||||
function register_assets() {
|
function register_assets() {
|
||||||
|
|
||||||
echo_debug "Registering the assets with the hpr database. register_assets()"
|
echo_debug "Registering the assets with the hpr database. register_assets()"
|
||||||
|
|
||||||
|
check_variable_is_correct working_dir ep_num
|
||||||
|
|
||||||
|
assets_csv="${working_dir}/hpr${ep_num}_assets.csv"
|
||||||
|
assets_json="${working_dir}/hpr${ep_num}_assets.json"
|
||||||
|
|
||||||
if [[ -s "${working_dir}/hpr${ep_num}_assets.csv" ]]
|
if [[ -s "${assets_csv}" ]]
|
||||||
then
|
then
|
||||||
echo_debug "Removing \"${working_dir}/hpr${ep_num}_assets.csv\"."
|
echo_debug "Removing \"${assets_csv}\"."
|
||||||
rm -v "${working_dir}/hpr${ep_num}_assets.csv"
|
rm -v "${assets_csv}"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
echo '"episode_id","filename","extension","size", "sha1sum", "mime_type", "file_type"' | tee "${working_dir}/hpr${ep_num}_assets.csv"
|
echo '"episode_id","filename","extension","size", "sha1sum", "mime_type", "file_type"' | tee "${assets_csv}"
|
||||||
|
|
||||||
for this_asset_filename in hpr${ep_num}.flac hpr${ep_num}.wav hpr${ep_num}.mp3 hpr${ep_num}.ogg hpr${ep_num}.opus hpr${ep_num}.srt hpr${ep_num}.txt $( find "${working_dir}/" -maxdepth 1 -type f -iname "hpr${ep_num}_image_*.*" )
|
for this_asset_filename in hpr${ep_num}.flac hpr${ep_num}.wav hpr${ep_num}.mp3 hpr${ep_num}.ogg hpr${ep_num}.opus hpr${ep_num}.srt hpr${ep_num}.txt $( find "${working_dir}/" -maxdepth 1 -type f -iname "hpr${ep_num}_image_*.*" )
|
||||||
do
|
do
|
||||||
@ -1454,21 +1480,21 @@ function register_assets() {
|
|||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
|
|
||||||
echo "${ep_num},\"${this_asset_basename}\",\"${this_asset_extension}\",\"${this_asset_size}\",\"${this_asset_sha1sum}\",\"${this_asset_mime_type}\",\"${this_asset_file_type}\"" | tee --append "${working_dir}/hpr${ep_num}_assets.csv"
|
echo "${ep_num},\"${this_asset_basename}\",\"${this_asset_extension}\",\"${this_asset_size}\",\"${this_asset_sha1sum}\",\"${this_asset_mime_type}\",\"${this_asset_file_type}\"" | tee --append "${assets_csv}"
|
||||||
|
|
||||||
done
|
done
|
||||||
|
|
||||||
if [ -s "${working_dir}/hpr${ep_num}_assets.csv" ]
|
if [ -s "${assets_csv}" ]
|
||||||
then
|
then
|
||||||
cat "${working_dir}/hpr${ep_num}_assets.csv" | csvtojson | jq '{"assets":[.[]]}' | tee "${working_dir}/hpr${ep_num}_assets.json"
|
cat "${assets_csv}" | csvtojson | jq '{"assets":[.[]]}' | tee "assets_json"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [ ! -s "${working_dir}/hpr${ep_num}_assets.json" ]
|
if [ ! -s "assets_json" ]
|
||||||
then
|
then
|
||||||
echo_error "The asset json file \"${working_dir}/hpr${ep_num}_assets.json\" is missing.";
|
echo_error "The asset json file \"assets_json\" is missing.";
|
||||||
fi
|
fi
|
||||||
|
|
||||||
response="$( curl --silent --netrc-file $HOME/.netrc --write-out '%{http_code}' --output /dev/null --request POST https://hub.hackerpublicradio.org/cms/assets.php --data-ascii @"${working_dir}/hpr${ep_num}_assets.json" --header "Content-Type: application/json" )"
|
response="$( curl --silent --netrc-file $HOME/.netrc --write-out '%{http_code}' --output /dev/null --request POST https://hub.hackerpublicradio.org/cms/assets.php --data-ascii @"assets_json" --header "Content-Type: application/json" )"
|
||||||
if [[ -z "${response}" || "${response}" != "200" ]]
|
if [[ -z "${response}" || "${response}" != "200" ]]
|
||||||
then
|
then
|
||||||
echo_error "The assets for episode hpr${ep_num} has not been registered. The response was \"${response}\""
|
echo_error "The assets for episode hpr${ep_num} has not been registered. The response was \"${response}\""
|
||||||
@ -1655,125 +1681,59 @@ function create_item_on_the_internet_archive() {
|
|||||||
# Send the derived files to the server borg to be sent to the Internet Archive
|
# Send the derived files to the server borg to be sent to the Internet Archive
|
||||||
|
|
||||||
function upload_files_to_the_internet_archive() {
|
function upload_files_to_the_internet_archive() {
|
||||||
|
|
||||||
echo_debug "Sending the derived files to Internet Archive. upload_files_to_the_internet_archive()"
|
echo_debug "Sending the derived files to Internet Archive. upload_files_to_the_internet_archive()"
|
||||||
|
|
||||||
check_variable_is_correct working_dir ep_num shownotes_html
|
check_variable_is_correct working_dir ep_num
|
||||||
|
|
||||||
# hpr4371.flac
|
if [ -z "${assets_json}" ]
|
||||||
# hpr4371.mp3
|
|
||||||
# hpr4371.ogg
|
|
||||||
# hpr4371.opus
|
|
||||||
# hpr4371_source.flac
|
|
||||||
# hpr4371.srt
|
|
||||||
# hpr4371.txt
|
|
||||||
# hpr4371.wav
|
|
||||||
#
|
|
||||||
|
|
||||||
if [ "$( find ${working_dir} -mindepth 1 -maxdepth 1 -type f \( -iname "hpr${ep_num}.flac" -or -iname "hpr${ep_num}.mp3" -or -iname "hpr${ep_num}.ogg" -or -iname "hpr${ep_num}.opus" -or -iname "hpr${ep_num}_source.*" -or -iname "hpr${ep_num}.srt" -or -iname "hpr${ep_num}.txt" -or -iname "hpr${ep_num}.wav" \) | wc --lines )" -ne "8" ]
|
|
||||||
then
|
then
|
||||||
echo_error "There are not 8 derived files for the Internet Archive."
|
assets_json="${working_dir}/episode_summary.json"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
while read this_ia_file
|
if [ ! -s "${assets_json}" ]
|
||||||
do
|
then
|
||||||
if [[ ! -s "${this_ia_file}" ]]
|
echo_debug "The \"assets_json\" variable/file is missing."
|
||||||
|
if [ "$( curl --silent --netrc --write-out '%{http_code}' https://hub.hackerpublicradio.org/cms/say.php?id=${ep_num} --output "${assets_json}" )" != 200 ]
|
||||||
then
|
then
|
||||||
ls -al "${this_ia_file}"
|
echo_error "The Episode hpr${ep_num} has not been posted"
|
||||||
echo_error "The derived files to the Internet Archive are missing \"${this_ia_file}\"."
|
|
||||||
fi
|
fi
|
||||||
done < <( find ${working_dir} -mindepth 1 -maxdepth 1 -type f \( -iname "hpr${ep_num}.flac" -or -iname "hpr${ep_num}.mp3" -or -iname "hpr${ep_num}.ogg" -or -iname "hpr${ep_num}.opus" -or -iname "hpr${ep_num}_source.*" -or -iname "hpr${ep_num}.srt" -or -iname "hpr${ep_num}.txt" -or -iname "hpr${ep_num}.wav" \) )
|
fi
|
||||||
|
check_variable_is_correct assets_json
|
||||||
|
|
||||||
|
|
||||||
#
|
|
||||||
# "${working_dir}/hpr${ep_num}.${extension}"
|
|
||||||
#
|
|
||||||
# ep_num="4371"
|
|
||||||
# ia upload hpr${ep_num} "${working_dir}/hpr${ep_num}.txt" --metadata=mediatype:audio --metadata="contributor:HackerPublicRadio" --metadata="creator:HPR Volunteers" --metadata="date:2025-05-05" --metadata="description:This show is a placeholder and will be updated soon." --metadata="language:eng" --metadata="licenseurl:http://creativecommons.org/licenses/by-sa/4.0" --metadata="title:A placeholder for hpr${ep_num}." --metadata=reviews-allowed:none --header x-archive-keep-old-version:0 --retries=5 --no-derive --no-backup
|
|
||||||
#
|
|
||||||
# wait for ia to finish
|
|
||||||
# ia tasks "hpr${ep_num}" | jq --slurp '[.[] | if .category == "catalog" then .status else empty end] | length'
|
|
||||||
|
|
||||||
# locate placeholder
|
# if [ "$( find ${working_dir} -mindepth 1 -maxdepth 1 -type f \( -iname "hpr${ep_num}.flac" -or -iname "hpr${ep_num}.mp3" -or -iname "hpr${ep_num}.ogg" -or -iname "hpr${ep_num}.opus" -or -iname "hpr${ep_num}_source.*" -or -iname "hpr${ep_num}.srt" -or -iname "hpr${ep_num}.txt" -or -iname "hpr${ep_num}.wav" \) | wc --lines )" -ne "8" ]
|
||||||
|
# then
|
||||||
# ia upload hpr4321_a686b0995e77c32d9e6566596ed9e4a2 /home/ken/tmp/hpr/processing/1739652162_4321_2025-02-24_624fd1a95db1d05e693eaf06e2f69b8667b0fc42edbc9/hpr4321.txt --metadata=mediatype:audio --metadata="collection:test_collection" --metadata="contributor:HackerPublicRadio" --metadata="creator:Klaatu" --metadata="date:2025-02-24" --metadata="description:This show is a placeholder and will be updated soon." --metadata="language:eng" --metadata="licenseurl:http://creativecommons.org/licenses/by-sa/4.0" --metadata="title:A placeholder for hpr4321." --metadata=reviews-allowed:none --header x-archive-keep-old-version:0 --retries=5 --no-derive --no-backup
|
# echo_error "There are not 8 derived files for the Internet Archive."
|
||||||
# locate placeholder
|
# fi
|
||||||
|
#
|
||||||
|
# while read this_ia_file
|
||||||
# for extension in .flac .mp3 .ogg .opus _source.ogg .srt .txt .wav
|
# do
|
||||||
# do
|
# if [[ ! -s "${this_ia_file}" ]]
|
||||||
# echo ia upload hpr${ep_num} "${working_dir}/hpr${ep_num}${extension}" --header "x-archive-keep-old-version:0" --retries=5 --no-derive --no-backup
|
# then
|
||||||
# done
|
# ls -al "${this_ia_file}"
|
||||||
#
|
# echo_error "The derived files to the Internet Archive are missing \"${this_ia_file}\"."
|
||||||
# ia upload hpr hpr4283_image_* --header "x-archive-keep-old-version:0" --retries=5 --no-derive --no-backup
|
# fi
|
||||||
|
#
|
||||||
|
# echo_debug "Sending \"${this_ia_file}\" to the Internet Archive"
|
||||||
# # # # for ext in flac mp3 ogg opus wav
|
#
|
||||||
# # # # do
|
#
|
||||||
# # # # if [[ ! -s "${working_dir}/hpr${ep_num}.${extension}" ]]
|
# # origin_sha1sum="$( echo $( ssh rsync.net "sha1 hpr/eps/hpr${ep_num}/${this_asset}" 2> /dev/null ) | awk '{print $NF}' )"
|
||||||
# # # # then
|
# # this_asset_sha1sum="$( sha1sum "${this_file}" | awk '{print $1}' )"
|
||||||
# # # # echo_error "The derived files to the IA are missing \"hpr${ep_num}.${extension}\"."
|
# #
|
||||||
# # # # ls -al "${working_dir}/hpr${ep_num}.${extension}"
|
# # if [[ -z "${origin_sha1sum}" || -z "${this_asset_sha1sum}" ]]
|
||||||
# # # # fi
|
# # then
|
||||||
# # # # done
|
# # echo_error "Could not determine the local/origin sha1sum for file \"${this_file}\"."
|
||||||
# # # #
|
# # fi
|
||||||
# # # # I don't know if you noticed in the documentation, but there are some key things you need to bear in mind when creating a new IA item:
|
# #
|
||||||
# # # #
|
# # ia download hpr_761c0f557b87090db3f8d4d9bce7fc70 hpr_761c0f557b87090db3f8d4d9bce7fc70_files.xml
|
||||||
# # # # You MUST upload the metadata first, and the way things seem to be organised, your item MUST have a file with it. That's why my test uploaded that 1 second WAV file. This is a constraint in the ia tool and was in the days of using the S3-like interface, so I imagine it's built-in to the IA software.
|
#
|
||||||
# # # # When creating the metadata you MUST define the mediatype, and since this controls some of what you can upload, using audio seems to be vital for HPR stuff.
|
# echo ia upload hpr${ep_num} "${this_ia_file}" --header "x-archive-keep-old-version:0" --retries=5 --no-derive --no-backup
|
||||||
# # # # I discovered (by accident) that if the metadata phase fails and the next phase (uploading files) continues to run, an item will be created but it will be a default thing which YOU CAN'T CHANGE! You have to plead with the people at info@archive.org to reset this item (they will never delete) because you can't change the defaults to the right values. If this happens and is not fixed, then HPR has effectively lost a slot (at least I tell myself this so that I don't inadvertently screw up an upload).
|
#
|
||||||
# # #
|
#
|
||||||
# # # ia upload hpr_761c0f557b87090db3f8d4d9bce7fc70 \
|
#
|
||||||
# # # test.wav \
|
# done < <( find ${working_dir} -mindepth 1 -maxdepth 1 -type f \( -iname "hpr${ep_num}.flac" -or -iname "hpr${ep_num}.mp3" -or -iname "hpr${ep_num}.ogg" -or -iname "hpr${ep_num}.opus" -or -iname "hpr${ep_num}_source.*" -or -iname "hpr${ep_num}.srt" -or -iname "hpr${ep_num}.txt" -or -iname "hpr${ep_num}.wav" \) )
|
||||||
# # # --metadata="mediatype:audio" \
|
|
||||||
# # # --metadata="collection:test_collection" \
|
|
||||||
# # # --metadata="collection:podcasts" \
|
|
||||||
# # # --metadata="contributor:HPR_test" \
|
|
||||||
# # # --metadata="creator:HPR_test" \
|
|
||||||
# # # --metadata="date:$(date +%F)" \
|
|
||||||
# # # --metadata="description:Summary: Uploading a test item from the command line<br />Source: We would point to the original show here<br /><br /><br /><br /> <p>This is some test HTML in a test item</p>" \
|
|
||||||
# # # --metadata="language:eng" \
|
|
||||||
# # # --metadata="licenseurl:http://creativecommons.org/licenses/by-sa/4.0" \
|
|
||||||
# # # --metadata="subject:Test" \
|
|
||||||
# # # --metadata="title:hpr_761c0f557b87090db3f8d4d9bce7fc70" \
|
|
||||||
# # # --header x-archive-keep-old-version:0 \
|
|
||||||
# # # --retries=5 \
|
|
||||||
# # # --no-derive \
|
|
||||||
# # # --no-backup
|
|
||||||
# # # # echo ia upload hpr4295 /data/IA/uploads/hpr4295.wav \
|
|
||||||
# # # # --metadata="mediatype:audio" \
|
|
||||||
# # # # --metadata="collection:hackerpublicradio" \
|
|
||||||
# # # # --metadata="collection:podcasts" \
|
|
||||||
# # # # --metadata="contributor:HackerPublicRadio" \
|
|
||||||
# # # # --metadata="creator:Ken Fallon" \
|
|
||||||
# # # # --metadata="date:2025-01-17" \
|
|
||||||
# # # # --metadata="description:Summary: Replacing the battery, swapping a fan, and getting a new desktop<br />Source: <a href="https://hackerpublicradio.org/eps/hpr4295/index.html">https://hackerpublicradio.org/eps/hpr4295/index.html</a><br /><br /><br /><br /><br /> <p> In this reserve show, Ken </p> <ol> <li> replaces the battery in a <a href="https://www.gsmarena.com/sony_xperia_10-9353.php" rel="noopener noreferrer" target="_blank"> SONY A10 telephone</a> . </li> <li> Swaps out a loud fan for a quiet one in a <a href="https://rigol.com.ua/en/products/digital-oscilloscope-rigol-ds1054z/" rel="noopener noreferrer" target="_blank"> RIGOL DS1054</a> . </li> <li> Then replaces the desktop of an <a href="https://www.ikea.com/us/en/p/bekant-desk-white-s19006323/" rel="noopener noreferrer" target="_blank"> Ikea BEKANT</a> standing desk with a narrower <a href="https://www.ikea.com/us/en/p/lagkapten-tabletop-white-40460815/" rel="noopener noreferrer" target="_blank"> LAGKAPTEN</a> table top not meant for it.</li> </ol>" \
|
|
||||||
# # # # --metadata="language:eng" \
|
|
||||||
# # # # --metadata="licenseurl:http://creativecommons.org/licenses/by-sa/4.0" \
|
|
||||||
# # # # --metadata="subject:IkeaHacks" \
|
|
||||||
# # # # --metadata="subject:diy" \
|
|
||||||
# # # # --metadata="subject:hardware" \
|
|
||||||
# # # # --metadata="title:hpr4295 :: Three Holiday Hacks from 2023" \
|
|
||||||
# # # # --header x-archive-keep-old-version:0 \
|
|
||||||
# # # # --retries=5 --no-derive --no-backup
|
|
||||||
# # # #
|
|
||||||
|
|
||||||
|
|
||||||
# ia upload hpr4295 /data/IA/uploads/hpr4295.wav \
|
|
||||||
# --metadata="mediatype:audio" \
|
|
||||||
# --metadata="collection:hackerpublicradio" \
|
|
||||||
# --metadata="collection:podcasts" \
|
|
||||||
# --metadata="contributor:HackerPublicRadio" \
|
|
||||||
# --metadata="creator:Ken Fallon" \
|
|
||||||
# --metadata="date:2025-01-17" \
|
|
||||||
# --metadata="description:Summary: Replacing the battery, swapping a fan, and getting a new desktop<br />Source: <a href="https://hackerpublicradio.org/eps/hpr4295/index.html">https://hackerpublicradio.org/eps/hpr4295/index.html</a><br /><br /><br /><br /><br /> <p> In this reserve show, Ken </p> <ol> <li> replaces the battery in a <a href="https://www.gsmarena.com/sony_xperia_10-9353.php" rel="noopener noreferrer" target="_blank"> SONY A10 telephone</a> . </li> <li> Swaps out a loud fan for a quiet one in a <a href="https://rigol.com.ua/en/products/digital-oscilloscope-rigol-ds1054z/" rel="noopener noreferrer" target="_blank"> RIGOL DS1054</a> . </li> <li> Then replaces the desktop of an <a href="https://www.ikea.com/us/en/p/bekant-desk-white-s19006323/" rel="noopener noreferrer" target="_blank"> Ikea BEKANT</a> standing desk with a narrower <a href="https://www.ikea.com/us/en/p/lagkapten-tabletop-white-40460815/" rel="noopener noreferrer" target="_blank"> LAGKAPTEN</a> table top not meant for it.</li> </ol>" \
|
|
||||||
# --metadata="language:eng" \
|
|
||||||
# --metadata="licenseurl:http://creativecommons.org/licenses/by-sa/4.0" \
|
|
||||||
# --metadata="subject:IkeaHacks" \
|
|
||||||
# --metadata="subject:diy" \
|
|
||||||
# --metadata="subject:hardware" \
|
|
||||||
# --metadata="title:hpr4295 :: Three Holiday Hacks from 2023" \
|
|
||||||
# --header x-archive-keep-old-version:0 \
|
|
||||||
# --retries=5 --no-derive --no-backup
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1857,9 +1817,9 @@ get_variables_from_episode_summary_json # Get the episode from HPR.
|
|||||||
|
|
||||||
# copy_derived_files_to_borg
|
# copy_derived_files_to_borg
|
||||||
|
|
||||||
create_item_on_the_internet_archive
|
# create_item_on_the_internet_archive
|
||||||
|
|
||||||
#upload_files_to_the_internet_archive
|
upload_files_to_the_internet_archive
|
||||||
|
|
||||||
#for i in {4301..4305};do echo ${i};/home/ken/sourcecode/personal/bin/hpr-check-ccdn-links.bash ${i};done
|
#for i in {4301..4305};do echo ${i};/home/ken/sourcecode/personal/bin/hpr-check-ccdn-links.bash ${i};done
|
||||||
|
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user