#!/bin/bash
# texmf_get.sh (c) 2016-2017 Johannes Schoepfer, Germany, slackbuilds[at]schoepfer[dot]info
# All rights reserved.
#
# Redistribution and use of this script, with or without modification, is
# permitted provided that the following conditions are met:
#
# 1. Redistributions of this script must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ''AS IS'' AND ANY EXPRESS OR IMPLIED
# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
# EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
# ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# V 0.9
#
# Prepare xz-compressed tarballs of texlive-texmf-trees based on texlive.tlpdb
# This script takes care of dependencies(as far as these are present in texlive.tlpdb) of collections and packages,
# and that every texlive-package is included only once.
#
# Policy from texlive-netarchive: Every package is included as dependency in exactly one collection. A package may have dependecies on other packages from any collection.
# packages source: http://mirror.ctan.org/systems/texlive/tlnet/archive/
#set -e
MAJORVERSION=2017
# release mirror
mirror="http://mirror.ctan.org/systems/texlive/tlnet/"
# pre-test mirror 2016
# mirror="http://ftp.cstug.cz/pub/tex/local/tlpretest/"
LANG=C
TMP=$PWD/tmp
output=$TMP/texlive.packages
output_doc=$TMP/texlive.doc.packages
errorlog=$TMP/error.log
texmf=$TMP/texmf
db=texlive.tlpdb
tmpfile=$(mktemp)
collections_done=$TMP/done
collections_tobedone=$TMP/tobedone
packages () {
# fonts-package first to make sure that big fonts like cm-super are not included elsewhere as dependency
PACKAGES="
cm-super
cbfonts
sanskrit-t1
uhc
fonts-tlwg
ethiop-t1
ipaex
wadalab
fandol
arphic
mxedruli
skaknew
padauk
japanese-otf
musixtex-fonts
unfonts-extra
baekmuk
arphic-ttf
unfonts-core
nanumtype1
" texmfget fonts
# put some stuff in "extra" to before it makes its way into base
PACKAGES="
arara
latex2nemeth
montex
" \
texmfget extra
# The base. some notes:
# cs needed by fmtutil-sys --all, 2017-06-24
PACKAGES="
$(cat $TMP/corepackages)
collection-basic
collection-latex
collection-latexrecommended
collection-xetex
collection-metapost
collection-fontutils
collection-luatex
collection-fontsrecommended
collection-humanities
collection-context
collection-mathscience
collection-plaingeneric
collection-binextra
yfonts
doublestroke
was
xypic
xindy
barcodes
qrcode
lastpage
datetime2
texdoc
appendix
changebar
footmisc
multirow
overpic
stmaryrd
subfigure
titlesec
siunitx
combelow
csquotes
etoolbox
etextools
glossaries
imakeidx
idxlayout
bidi
filecontents
eplain
texsis
mltex
lollipop
moreverb
collection-langcyrillic
collection-langeuropean
collection-langenglish
collection-langfrench
collection-langgerman
collection-langgreek
collection-langitalian
collection-langpolish
collection-langportuguese
collection-langspanish
" texmfget base
# Call "fonts"-tarball again to add remaining fonts
PACKAGES="
collection-fontsextra
" texmfget fonts
# Put all remaining stuff in the "extra" tarball
PACKAGES="
biber.x86_64-linux
biber.i386-linux
collection-formatsextra
collection-latexextra
collection-pictures
collection-games
collection-publishers
collection-bibtexextra
collection-music
collection-pstricks
collection-texworks
collection-wintools
collection-langczechslovak
collection-langjapanese
collection-langkorean
collection-langarabic
collection-langchinese
collection-langcjk
collection-langother
" texmfget extra
# The docs-tarball - very big (about 1300 MB)
texmfget docs
if [ "$TARBALL" != docs ]
then
echo "Packages-list: $output.meta.$TARBALL"
while read collection
do
grep -w "$collection" $collections_done &> /dev/null
[ $? != 0 ] && echo "WARNING: $collection was not processed, please edit packages-function."
done < $TMP/allcollections
fi
}
# ==== Nothing to edit beyond this line (hopefully) ====
usage () {
echo "Prepare texmf trees based on collections and packages and their dependencies."
echo "./texmf_get.sh [base|docs|extra|fonts]"
exit
}
package_meta () {
echo "collection/package $collection"
# collection start linenumer
start_n="$(grep -n ^"name ${collection}$" $TMP/$db | cut -d':' -f1)"
[ -z "$start_n" ] && echo "$collection was not found in $TMP/$db, bye." && exit 1
# find end of package/collection
for emptyline in $emptylines
do
if [ "$emptyline" -gt "$start_n" ]
then
end_n=$emptyline
break
fi
done
sed "${start_n},${end_n}!d" $TMP/$db > $tmpfile
}
package_list () {
# Remove outputfile if already present
[ -s "$output" ] && rm $output
# Only do something if $collection wasn't already done before
while [ -s $collections_tobedone ]
do
collection=$(tail -n1 $collections_tobedone)
# continue with next collection if collection was already done
if [ -s "$collections_done" ]
then
grep -w "^${collection}$" $collections_done &> /dev/null
if [ $? = 0 ]
then
sed -i "/^$collection$/d" $collections_tobedone
continue
fi
fi
if [ -s "$texmf/$collection.meta" ]
then
cp $texmf/$collection.meta $tmpfile
else
package_meta
fi
# Don't handle collections as dependency of other collections, as this adds far too much and therefore reduces controll over what packages to be added
sed -i "/^depend collection/d" $tmpfile
# If $collection is a singel package, add it here
if [ -n "$(head -n1 $tmpfile | fgrep -v "name collection" )" ]
then
packagelist="$TMP/run.tlpkg"
if [ -z "$(grep -w "^${collection}$" $packagelist)" ]
then
sed -i "/^$collection$/d" $collections_tobedone
echo "$collection" >> $collections_done
continue
fi
echo "$collection" >> $output
fi
# add dependend packages
grep ^"depend " $tmpfile | grep -v "ARCH$" | cut -d' ' -f2- >> $collections_tobedone
echo "$collection" >> $collections_done
sed -i "/^${collection}$/d" $collections_tobedone
done
}
untar () {
# Download packages, if not already available. Not every packages has a corresponding .doc package.
[ -f $output.meta ] && rm $output.meta
while read package
do
# untar all packages, check for relocation, "relocate 1" -> untar in texmf-dist
if [ "$flavour" = ".doc" ]
then
sha512="$(grep ^doccontainerchecksum $texmf/$package.meta | cut -d' ' -f2 )"
else
sha512="$(grep ^containerchecksum $texmf/$package.meta | cut -d' ' -f2 )"
fi
# try two times if package isn't present, with -t1 to get another mirror the second time
[ ! -s ${package}${flavour}.tar.xz ] && wget -t1 -c ${mirror}archive/${package}${flavour}.tar.xz
[ ! -s ${package}${flavour}.tar.xz ] && wget -t1 -c ${mirror}archive/${package}${flavour}.tar.xz
[ ! -s ${package}${flavour}.tar.xz ] && echo "Downloading ${package}${flavour}.tar.xz did not work, writing to $errorlog" && echo "Error downloading ${package}${flavour}.tar.xz" >> $errorlog && exit 1
# check sha512, give three tries for downloading aggain(diffrent mirrors are used automatically)
for tillthree in 1 2 3
do
if [ "$(sha512sum ${package}${flavour}.tar.xz | cut -d' ' -f1 )" != "$sha512" ]
then
# Download (hopefully) newer file
rm ${package}${flavour}.tar.xz
wget -t1 -c ${mirror}archive/${package}${flavour}.tar.xz
else
break
fi
done
# check sha512 again, exit if it fails
if [ "$(sha512sum ${package}${flavour}.tar.xz | cut -d' ' -f1 )" != "$sha512" ]
then
echo "sha512sum $(sha512sum ${package}${flavour}.tar.xz | cut -d' ' -f1 ) of"
echo "${package}${flavour}.tar.xz doesn't match with $TMP/$db"
echo "sha512sum $sha512"
echo "Delete $TMP/${db}* and try again."
exit 1
fi
# Exclude the tlpkg-stuff, TLUtils.pm(needed tu run texlive) is provided by texlive-source
grep -w ^"relocated 1" $texmf/$package.meta &>/dev/null
if [ $? = 0 ]
then
tar vxf ${package}${flavour}.tar.xz --exclude tlpkg -C texmf-dist || exit 1
else
tar vxf ${package}${flavour}.tar.xz --exclude tlpkg || exit 1
fi
# In case a binary package was decompressed, put it in texmf-dist
[ -d bin ] && cp -a bin texmf-dist && rm -rf bin
if [ "$flavour" = ".doc" ]
then
size=$(( $(grep ^doccontainersize $texmf/$package.meta | cut -d' ' -f2 ) / 1024 ))
else
size=$(( $(grep ^containersize $texmf/$package.meta | cut -d' ' -f2 ) / 1024 ))
fi
shortdesc="$(grep ^shortdesc $texmf/$package.meta | cut -d' ' -f2- )"
echo "$size Kb, $package: $shortdesc" >> $output.meta
done < $1
# Copy packages-list to texmf-dist, so included packages are known in later installation
sort -n $output.meta > $output.meta.$TARBALL
cp $output.meta.$TARBALL texmf-dist/
rm $output.meta
}
remove_cruft () {
# Remove m$-stuff, ConTeXt single-user-system stuff, source leftovers and pdf-versions of manpages
rm -rf texmf-dist/source
rm -rf texmf-dist/scripts/context/stubs/source/
find texmf-dist/ -type d -name 'win32' -exec rm -rf {} +
find texmf-dist/ -type d -name 'win64' -exec rm -rf {} +
find texmf-dist/ -type d -name 'mswin' -exec rm -rf {} +
find texmf-dist/ -type d -name 'win' -exec rm -rf {} +
find texmf-dist/ -type d -name 'setup' -exec rm -rf {} +
find texmf-dist/ -type d -name 'install' -exec rm -rf {} +
find texmf-dist/ -type f -name '*.bat' -delete
find texmf-dist/ -type f -name '*win32*' -delete
find texmf-dist/ -type f -name 'winansi*' -delete
find texmf-dist/ -type f -name '*-man.pdf' -delete
# Remove zero-length files, as these appear e.g. in hyph-utf8 tex-package.
find . -type f -size 0c -delete
}
texmfget () {
# Check all content to make sure no package is added more than once. Docs contain every docfile
if [ $TARBALL != docs ]
then
echo "Preparing list of packages to be added to ${1}-tarball ..."
echo "$PACKAGES" | sed "s/[[:space:]]//g;/^$/d" >> $collections_tobedone
package_list
fi
# Process only the one named($1) tarball
if [ "$1" = $TARBALL ]
then
cd $texmf
# Cleanup tar-directory, just in case
[ -d texmf-dist ] && rm -rf texmf-dist
mkdir texmf-dist &> /dev/null
# Make tarball/checksum reproducible by setting mtime(clamp-mtime), owner, group and sort content
# Doesn't work with tar 1.13, when makepkg creates the tarball:
# tar-1.13: time_t value 9223372036854775808 too large (max=68719476735)
VERSION=$(cat $TMP/VERSION)
case $TARBALL in
docs)
export flavour=".doc"
untar $output_doc
remove_cruft
#tar vrf $TMP/texlive-$TARBALL-$VERSION.tar --clamp-mtime --mtime --owner=0 --group=0 --sort=name texmf-dist || exit 1
tar vrf $TMP/texlive-$TARBALL-$VERSION.tar --owner=0 --group=0 --sort=name texmf-dist || exit 1
echo "Packages-list: $output_doc"
rm -rf texmf-dist
;;
base|extra|fonts)
untar $output
remove_cruft
#tar vrf $TMP/texlive-$TARBALL-$VERSION.tar --clamp-mtime --mtime --owner=0 --group=0 --sort=name texmf-dist || exit 1
tar vrf $TMP/texlive-$TARBALL-$VERSION.tar --owner=0 --group=0 --sort=name texmf-dist || exit 1
rm -rf texmf-dist
;;
esac
fi
}
# Main
case "$1" in
base|docs|extra|fonts) TARBALL=$1; echo "Building $TARBALL tarball ..." ;;
*) usage ;;
esac
mkdir -p $texmf
cd $TMP
# Create run.tlpkg and doc.tlpkg only if $db.orig isn't there yet/was deleted
if [ ! -s $TMP/${db}.orig -o ! -s $TMP/${db} ]
then
echo $MAJORVERSION.$(date +%y%m%d) > VERSION
wget -c -O $TMP/${db}.orig -c ${mirror}tlpkg/$db
# shrink db to be faster on later processing
sed "/^ \+./d;/^longdesc \+./d;/^rev\+./d;/^exe\+./d;/^bin\+./d;/^src\+./d" $TMP/${db}.orig > $TMP/$db
# As $db (might be)is new, remove the meta-files, might created again with (pontentionally) new content
rm -rf $texmf/*.meta
rm $TMP/run.tlpkg
[ -f "$output_doc" ] && rm "$output_doc"
# Make a list of all packages available, but exclude binary and installer/configuration packages.
# It turns out that packagenames without '.' are what we want. Packages with '.' are all binarie-packages, which are build from source.
grep ^name $TMP/$db | grep -v ^"name collection-" | grep -v ^"name scheme-" | grep -v '\.' | cut -d' ' -f2 > $TMP/allpackages
# Make a list of all collections
grep ^"name collection-" $TMP/$db | cut -d' ' -f2 > $TMP/allcollections
# Provide TLCore packages for the base-tarball, as these packages(and their dependencies) should be present in any case.
grep -B1 ^'category TLCore' $TMP/$db | grep -v ^'category TLCore' | grep -v ^-- | grep -v '\.' | cut -d' ' -f2 > $TMP/corepackages
# add biber (perl)binaries as special exception as it is not fun to build
cat <<- EOF >> $TMP/allpackages
biber.x86_64-linux
biber.i386-linux
EOF
fi
# globaly excluded packages, which does not make sense without tlpkg-installer, or are non-linux specific, or are already covered by the sourcebuild,
# or are covered by an external package(asymptote), or obsolete packages(datetime replaced by datetime2, anysize replaced by geometry)
cat << EOF > $TMP/global_exclude
tlcockpit
tlshell
texosquery
asymptote
asymptote-by-example-zh-cn
asymptote-faq-zh-cn
asymptote-manual-zh-cn
EOF
while read exlude
do
sed -i "/^${i}$/d" $TMP/allpackages
[ -s $TMP/run.tlpkg ] && sed -i "/^${exclude}$/d" $TMP/run.tlpkg
[ -s $output_doc ] && sed -i "/^${exclude}$/d" $output_doc
done < $TMP/global_exclude
# Get linenumbers of empty lines
[ -z "$emptylines" ] && emptylines="$(grep -n ^$ $TMP/$db | cut -d':' -f1)"
# Sort doc- and run- packages out to avoid binfiles and sourcfile in the texmf-tree
while read collection
do
if [ ! -s $texmf/$collection.meta ]
then
package_meta
grep ^runfiles $tmpfile &>/dev/null
if [ $? = 0 ]
then
echo $collection >> $TMP/run.tlpkg
fi
grep ^docfiles $tmpfile &>/dev/null && echo $collection >> $output_doc
mv $tmpfile $texmf/$collection.meta
fi
done < $TMP/allpackages
# Handle biber binaries to be add-able
cat << EOF >> $TMP/run.tlpkg
biber.x86_64-linux
biber.i386-linux
EOF
[ -f "$collections_done" ] && rm "$collections_done"
[ -f "$collections_tobedone" ] && rm "$collections_tobedone"
packages
# Now the demanded packages are in the tarball, compress it.
echo "Compressing $TMP/texlive-$TARBALL-$VERSION.tar ..."
if [ -s $TMP/texlive-$TARBALL-$VERSION.tar ]
then
[ -f $TMP/texlive-$TARBALL-$VERSION.tar.xz ] && rm $TMP/texlive-$TARBALL-$VERSION.tar.xz
xz -9 -T0 $TMP/texlive-$TARBALL-$VERSION.tar || exit 1
md5sum $TMP/texlive-$TARBALL-$VERSION.tar.xz
ls -lah $TMP/texlive-$TARBALL-$VERSION.tar.xz
fi
# cleanup
rm $tmpfile
|
