\documentclass[pdf]{prosper} \usepackage[toc,highlight,linbit,notes,hlsections]{HA-prosper} \title{ROCK Linux} \subtitle{The Toolkit for Building Linux Distributions} \author{Clifford Wolf - www.clifford.at} \DefaultTransition{Wipe} \TitleSlideNav{FullScreen} \NormalSlideNav{ShowBookmarks} \LeftFoot{\href{http://www.clifford.at}{Clifford Wolf}, \today} \RightFoot{\href{http://www.rocklinux.org/}{ROCK Linux - www.rocklinux.org}} \begin{document} \maketitle % ============================================================================ \tsectionandpart{Introduction} \begin{slide}{The Problem} \begin{itemize} \item Building a Linux distribution is complex and time-consuming \begin{itemize} \item Even small distributions have a few hundred packages \item Bootstrapping a distribution can take a few weeks if it is done cleanly \item Crossbuilding can be very tricky \end{itemize} \vspace*{.5cm} \item Keeping it up-to-date is even more time consuming \begin{itemize} \item All packages need to be monitored for updates \item Updates must be tested and checked for regressions \item Most regressions appear in the depending packages \end{itemize} \vspace*{.5cm} \item All distributions have a lot in common \begin{itemize} \item The packages themselves are almost the same \item Package configurations and init scripts are very similar \end{itemize} \end{itemize} \end{slide} \begin{slide}{The Solution} \begin{itemize} \item ROCK Linux makes distribution development easy: \vspace*{.5cm} \begin{verbatim} ./scripts/Config -cfg mydist ./scripts/Config -cfg myboot ./scripts/Download -cfg mydist -required ./scripts/Download -cfg myboot -required ./scripts/Build-Target -cfg mydist ./scripts/Build-Target -cfg myboot ./scripts/Create-ISO demo myboot mydist \end{verbatim} \vspace*{.5cm} \item Only the features specific to the new distribution need to be added. \vspace*{.5cm} \item ROCK Linux makes easy things easy and complex things possible. \end{itemize} \end{slide} % ============================================================================ \tsectionandpart{Config Tool} \begin{slide}{Abstract} \begin{itemize} \item The ./scripts/Config tool is used to configure the to be built system \vspace*{.5cm} \item The available config options are defined in config.in files \vspace*{.5cm} \item The config.in files don't need to be specially registered with ./scripts/Config \vspace*{.5cm} \item The config script is just a big shell script, the config.in files add additional calls to special functions to alter the config variables. \end{itemize} \end{slide} \begin{slide}{Call-Tree (1/2)} \begin{verbatim} Execution of sub-scripts: - achitecture/*/preconfig.in * Selecting Architecture * architecture/$ROCKCFG_ARCH/config.in - misc/*/preconfig.in - target/*/preconfig.in - package/*/*/preconfig.in * Selecting Target * target/$ROCKCFG_TARGET/config.in * misc/*/noexpertconfig.in Only procedures marked with '*' interact with the user (register options). \end{verbatim} \end{slide} \begin{slide}{Call-Tree (2/2)} \begin{verbatim} Execution of sub-scripts (continued): * misc/*/noexpertconfig.in * {package/*,misc}/*/config-*.in * {package/*,misc}/*/config.in * Various common build options - package/*/*/postconfig.in - misc/*/postconfig.in - architecture/$ROCKCFG_ARCH/postconfig.in - target/$ROCKCFG_TARGET/postconfig.in Only procedures marked with '*' might interact with the user (register options). \end{verbatim} \end{slide} \begin{slide}{Namespaces} \begin{verbatim} Naming-scheme for extending config variables: Core: ROCKCFG_* Archs: ROCKCFG_ARCH__* Targets: ROCKCFG_TRG__* Packages: ROCKCFG_PKG__* Config-Internal Variables: Core: CFGTEMP_* Archs: CFGTEMP_ARCH__* Targets: CFGTEMP_TRG__* Packages: CFGTEMP_PKG__* Config Presets: ROCKCFGSET_* \end{verbatim} \end{slide} \begin{slide}{Example config.in} \begin{verbatim} if pkgcheck subversion X then menu_begin MENU_PKG_SUBVERSION 'Subversion Options' bool 'Create a subversion-static package' \ ROCKCFG_PKG_SUBVERSION_STATIC 1 if [ "$ROCKCFG_PKG_SUBVERSION_STATIC" = "1" ] then pkgfork subversion subversion-static fi menu_end fi \end{verbatim} \end{slide} % ============================================================================ \tsectionandpart{Packages} \begin{slide}{Abstract} \begin{itemize} \item Every package has a subdirectory in the package/ directory. \\ E.g.: package/base/gcc \vspace*{.5cm} \item Every package has a *.desc file with the metadata \vspace*{.5cm} \item A semi-intelligent system autodetects how packages should be built \vspace*{.5cm} \item Only some packages need dedicated build instructions in a *.conf file \vspace*{.5cm} \item Patches (*.patch files in the package dir) are applied automatically \vspace*{.5cm} \item So creating patches is extremely easy with ROCK Linux \end{itemize} \end{slide} \begin{slide}{Repositories} \begin{itemize} \item The packages are grouped in so-called repositories \vspace*{.5cm} \item Every developer or developer group has their own repository \vspace*{.5cm} \item E.g.: "base" for the core group \\ or "clifford" for Clifford Wolf \vspace*{.5cm} \item This is completely independent from the package categories \vspace*{.5cm} \item There also is e.g. a "kde" repository for the kde base packages. That's to make switching maintainership for this packages easy and should not be confused with the "desktop/kde" package category. \end{itemize} \end{slide} \begin{slide}{Example *.desc file} \begin{verbatim} [I] System-wide profiler for Linux systems [T] OProfile is a system-wide profiler for Linux [T] systems, capable of <...> [A] John Levon [M] Clifford Wolf [C] base/system [L] GPL [S] Stable [V] 0.8.1 [P] X -?---5---9 145.500 [D] 3487496302 oprofile-0.8.1.tar.gz http://dl.sourceforge.net/sourceforge/oprofile/ \end{verbatim} \end{slide} \begin{slide}{Example *.conf file} \begin{verbatim} install_pulpstone() { cp -v $confdir/pulpstoner.pl \ $root/usr/bin/pulpstoner cp -v $confdir/pulpstonec.pl \ $root/usr/bin/pulpstonec cp -v $confdir/pulpstoned.sh \ $root/usr/bin/pulpstoned chmod +x $root/usr/bin/pulpstone[rcd] } hook_add postmake 5 "install_pulpstone" confopt="$confopt --with-kernel-support" \end{verbatim} \end{slide} % ============================================================================ \tsectionandpart{Architectures} \begin{slide}{Abstract} \begin{itemize} \item Every architecture has a subdirectory in the architecture/ directory. \vspace*{.5cm} \item The architecture attributes are described by an archtest.out file. \vspace*{.5cm} \item More complex architectures (with sub-architectures with e.g. multiple endianes), an archtest.sh can be used to hold the dynamic data. \vspace*{.5cm} \item Various sub-systems may expect additional files. \\ E.g. gcc is looking for a "gcc-options" file \end{itemize} \end{slide} \begin{slide}{Example archtest.out file} \begin{verbatim} arch_sizeof_short=2 arch_sizeof_int=4 arch_sizeof_long=4 arch_sizeof_long_long=8 arch_bigendian=yes \end{verbatim} \end{slide} \begin{slide}{Example archtest.sh file} \begin{verbatim} case "$ROCKCFG_HPPA_BITS" in 32) arch_machine=hppa arch_target="hppa-unknown-linux-gnu" arch_sizeof_char_p=4 ;; 64) arch_machine=hppa arch_target="hppa64-unknown-linux-gnu" arch_sizeof_char_p=8 ;; esac \end{verbatim} \end{slide} \begin{slide}{Example gcc-options file} \begin{verbatim} if [ "$ROCKCFG_SPARC_OPT" != "generic" ] ; then var_append GCC_WRAPPER_INSERT \ " " "-mcpu=$ROCKCFG_SPARC_OPT" tune="" case "$ROCKCFG_SPARC_OPT" in v7) tune=cypress ;; v8) tune=supersparc ;; v9) tune=ultrasparc ;; esac [ -n "$tune" ] && var_append \ GCC_WRAPPER_INSERT " " "-mtune=$tune" if [ $ROCKCFG_SPARC_BITS == 64 ] ; then var_append GCC_WRAPPER_INSERT " " \ "-Wa,-Av9a -mno-app-regs" fi fi \end{verbatim} \end{slide} % ============================================================================ \tsectionandpart{Targets} \begin{slide}{Abstract} \begin{itemize} \item A ROCK Linux based distribution is called a "target" \vspace*{.5cm} \item An installable CD-set usually consists of two targets: \begin{itemize} \item A small boot-from-cd installer distribution \item The distribution to be installed on the system \end{itemize} \vspace*{.5cm} \item Each target has its own configuration (-cfg option) \end{itemize} \end{slide} \begin{slide}{Example config code} \begin{verbatim} CFGTEMP_TARGETLIST="$CFGTEMP_TARGETLIST crystal \ Crystal_ROCK_(The_general_purpose_distriution)" pkgfilter sed -e '/CORE/ ! s/^X/O/' ROCKCFGSET_DO_REBUILD_STAGE=0 \end{verbatim} \end{slide} \begin{slide}{Example build code} {\tiny \begin{verbatim} pkgloop echo_header "Finishing build." echo_status "Creating package database ..." admdir="build/${ROCKCFG_ID}/var/adm" create_package_db $admdir build/${ROCKCFG_ID}/ROCK/pkgs \ build/${ROCKCFG_ID}/ROCK/pkgs/packages.db echo_status "Creating isofs.txt file .." cat << EOT > build/${ROCKCFG_ID}/ROCK/isofs.txt DISK1 $admdir/cache/ ${ROCKCFG_SHORTID}/info/cache/ DISK1 $admdir/cksums/ ${ROCKCFG_SHORTID}/info/cksums/ DISK1 $admdir/dependencies/ ${ROCKCFG_SHORTID}/info/dependencies/ DISK1 $admdir/descs/ ${ROCKCFG_SHORTID}/info/descs/ DISK1 $admdir/flists/ ${ROCKCFG_SHORTID}/info/flists/ DISK1 $admdir/md5sums/ ${ROCKCFG_SHORTID}/info/md5sums/ DISK1 $admdir/packages/ ${ROCKCFG_SHORTID}/info/packages/ EVERY build/${ROCKCFG_ID}/ROCK/pkgs/packages.db ${ROCKCFG_SHORTID}/pkgs/packages.db SPLIT build/${ROCKCFG_ID}/ROCK/pkgs/ ${ROCKCFG_SHORTID}/pkgs/ EOT \end{verbatim} } \end{slide} % ============================================================================ \tsectionandpart{Wrappers} \begin{slide}{Filelist Wrapper} \begin{itemize} \item Managing the package file list by hand is a waste of time \vspace*{.5cm} \item ROCK Linux is using a so-called "flist wrapper" to monitor package build processes and create file lists automatically. \vspace*{.5cm} \item The same mechanism is also used for auto-detecting build-time dependencies. %\vspace*{.5cm} %\item The same mechanism is also used for auto-detecting build-time dependencies. \end{itemize} \end{slide} \begin{slide}{Command Wrapper (1/3)} \begin{itemize} \item Patching Makefiles to honour \$CFLAGS or install files in different locations can be a pain in the neck. \vspace*{.5cm} \item So ROCK Linux has the command wrapper infrastructure to dynamically rewrite commands executed by the build process. \vspace*{.5cm} \item The common command wrapper is primarly used for mangling gcc options. \vspace*{.5cm} \item Special wrappers exist for commands which are used to install files, such as cp and install. \end{itemize} \end{slide} \begin{slide}{Command Wrapper (2/3)} GCC Option Wrapper Examples: \begin{verbatim} [...] if [ $pkg != glibc23 -a $pkg != glibc22 -a \ $pkg != grub -a $pkg != dietlibc ] then if [ "$ROCKCFG_PKG_GCC_STACKPRO" = 1 ] ; then var_append GCC2_WRAPPER_INSERT \ " " "-fstack-protector" var_append GCC3_WRAPPER_INSERT \ " " "-fstack-protector" fi fi [...] \end{verbatim} \end{slide} \begin{slide}{Command Wrapper (3/3)} Install Wrapper Examples: \begin{verbatim} [...] var_append INSTALL_WRAPPER_FILTER "|" \ "sed -e 's,usr/bin/find,bin/find,' \ -e 's,usr/bin/xargs,bin/xargs,'" [...] [...] var_append INSTALL_WRAPPER_FILTER "|" \ "sed 's,man1/xemacs\.1$,man1/xemacs-beta.1,'" [...] \end{verbatim} \end{slide} % ============================================================================ \tsectionandpart{Some Other Features} \begin{slide}{Pseudo-Native Builds} \begin{itemize} \item Doing native builds isn't usefull on some architectures (such as old Motorola 68000 systems). \vspace*{.5cm} \item But cross-building is sometimes not possible (broken configure scripts and makefiles, etc). \vspace*{.5cm} \item With "pseudo-native" builds it's possible to do a build which looks like a native build on an alien architecture. \vspace*{.5cm} \item This is done by registering a small wrapper binary as handler for ELF binaries from the target architecture with the Linux kernel binfmt\_misc hook. \vspace*{.5cm} \item This wrapper either calls a local replacement executable on the host architecture or (in the remaining cases) runs the program on another machine of the target architecture. \end{itemize} \end{slide} \begin{slide}{Package Forks and Splits (1/2)} \begin{itemize} \item One package description may describe the build process for more than one package. E.g. there is only one gcc package in ROCK Linux which currently handles the packages gcc2, gcc32, gcc33 and gcc34. \vspace*{.5cm} \item This is possible due to a mechanism called "package forking". \vspace*{.8cm} \item One package build process may produce more than one binary package. E.g. building the subversion package ends in the packages subversion, subversion:doc, subversion:dev, subversion:server and subversion:apache. \vspace*{.5cm} \item This is possible due to a mechanism called "package splitting". \end{itemize} \end{slide} \begin{slide}{Package Forks and Splits (2/2)} Package Split Example Code: \begin{verbatim} [...] if pkginstalled apache; then apacheprefix=${ROCKCFG_PKG_APACHE_PREFIX:-opt/apache} [...] splitdesc_apache() { desc_I="Apache module for subversion" } splitreg 50 apache ${apacheprefix}.*mod_.*so fi [...] \end{verbatim} \end{slide} \begin{slide}{Output Modules} \begin{itemize} \item The output of the build process can be passed through various output modules. \vspace*{.5cm} \item One of those output modules are used for the "normal" status output on the terminal. \vspace*{.5cm} \item Some examples for other output modules are the modules for speech synthesis, html pages and mythtvosd. \vspace*{.5cm} \item It's possible to use some ouput modules only for specific events. \\ E.g. only using audio ouput using speech synthesis on errors. \end{itemize} \end{slide} \begin{slide}{Crystal ROCK (1/2)} \begin{itemize} \item Most users don't really want to build their own distributions \vspace*{.5cm} \item So ROCK Linux 3.0 will also contain a general purpose distribution called "Crystal ROCK". \vspace*{.5cm} \item Crystal ROCK will be small (one binary CD for x86), 200-300 packages. \vspace*{.5cm} \item It contains the most common desktop packages (such as KDE and all it's utilities) development tools (like gcc) and servers (like apache). \end{itemize} \end{slide} \begin{slide}{Crystal ROCK (2/2)} \begin{itemize} \vspace*{.5cm} \item Missing packages can be installed from binary package archives from the internet or built from source using the "rocket" tool. \vspace*{.5cm} \item The "stone" program provides users with an easy-to-use config tool, but all the config files may also be edited directly and and primarly designed to be edited using a random ASCII editor. \vspace*{.5cm} \item An alternative graphical installer is planed for the future. \vspace*{.5cm} \item The Crystal ROCK target may be a good starting point for your own distributions. \end{itemize} \end{slide} % ============================================================================ \tsectionandpart{URLs and References} \begin{slide}{URLs} \begin{itemize} \item ROCK Linux Webpage \\ http://www.rocklinux.org/ \vspace*{.5cm} \item ROCK Linux Portal \\ http://www.rocklinux.net/ \vspace*{.8cm} \item Mailing Lists \\ http://www.rocklinux.org/mail.html \vspace*{.5cm} \item IRC Channel \\ http://www.rocklinux.org/irc.html \end{itemize} \end{slide} \begin{slide}{Credits} \begin{itemize} \item ROCK Linux Team \\ http://www.rocklinux.org/gallery.html \vspace*{.5cm} \item Clifford Wolf: \\ http://www.clifford.at/ \vspace*{.5cm} \item LINBIT Information Technologies \\ http://www.linbit.com/ \end{itemize} \end{slide} % ============================================================================ \end{document}