\documentclass[a4paper]{article}
\usepackage{ascmac,graphicx,epic,eepic,itembbox}
\setlength{\topmargin}{-0.8cm}
\setlength{\oddsidemargin}{0.0cm}
\setlength{\textwidth}{16cm}
\setlength{\textheight}{22.4cm}
 \newcommand{\R}{\mbox{\boldmath $R$}}
 \newcommand{\vx}{\mbox{\boldmath $x$}}
 \newcommand{\vy}{\mbox{\boldmath $y$}}
 \newcommand{\va}{\mbox{\boldmath $a$}}
 \newcommand{\vr}{\mbox{\boldmath $r$}}
 \newcommand{\vf}{\mbox{\boldmath $f$}}
 \newcommand{\vb}{\mbox{\boldmath $b$}}
 \newcommand{\vc}{\mbox{\boldmath $c$}}
 \newcommand{\vu}{\mbox{\boldmath $u$}}
 \newcommand{\vz}{\mbox{\boldmath $z$}}
 \newcommand{\ve}{\mbox{\boldmath $e$}}
 \newcommand{\vv}{\mbox{\boldmath $v$}}
 \newcommand{\vzero}{\mbox{\boldmath $0$}}
 \newcommand{\vd}{\mbox{\boldmath $d$}}
 \newcommand{\vxi}{\mbox{\boldmath $\xi$}}
 \newcommand{\vZ}{\mbox{\boldmath $Z$}^{n \times n}}
 \newcommand{\vepsilon}{\mbox{\boldmath $\varepsilon$}}
\newcommand{\namelistlabel}[1]{\mbox{#1}\hfill}
\newenvironment{namelist}[1]{%
 \begin{list}{}
  {\let\makelabel\namelistlabel
  \settowidth{\labelwidth}{#1}
  \setlength{\leftmargin}{1.1\labelwidth}}
}{%
\end{list}}
\makeatletter
\@addtoreset{equation}{section}
\def\theequation{\thesection.\arabic{equation}}
\makeatother
\title{Lis User Guide Version 1.4.12}
\author{}
\date{}
\begin{document}

\vspace*{4cm}
\begin{flushleft}
{\Large Lis User Guide}\\
Version 1.4.12
\end{flushleft}

\vspace*{2cm}
\begin{figure}[h]
\includegraphics[scale=0.7]{irises_korin.eps}
%\includegraphics[scale=0.3]{shobuen_hiroshige.eps}
\end{figure}

\vspace*{2cm}
\begin{flushleft}
{\large The Scalable Software Infrastructure
Project\\
{\tt http://www.ssisc.org/}}\\
\end{flushleft}

\vspace*{5mm}
\thispagestyle{empty}
\newpage
\begin{flushleft}
{\small
Copyright \copyright\ The Scalable Software Infrastructure Project,
supported by ``Development of Software Infrastructure for Large Scale
Scientific Simulation'' Team, CREST, JST.

Akira Nishida, Research Institute for Information Technology, 
Kyushu University, 6-10-1, Hakozaki, Higashi-ku, Fukuoka 812-8581, Japan.

All rights reserved.

\vspace*{5mm}
 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions are
 met:

 1. Redistributions of source code must retain the above copyright
 notice, this list of conditions and the following disclaimer.

 2. Redistributions in binary form must reproduce the above copyright
 notice, this list of conditions and the following disclaimer in the
 documentation and/or other materials provided with the distribution.

 3. Neither the name of the University nor the names of its contributors
 may be used to endorse or promote products derived from this software
 without specific prior written permission.

\vspace*{5mm}
 THIS SOFTWARE IS PROVIDED BY THE SCALABLE SOFTWARE INFRASTRUCTURE
 PROJECT ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT
 NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE SCALABLE
 SOFTWARE INFRASTRUCTURE PROJECT BE LIABLE FOR ANY DIRECT, INDIRECT,
 INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
 BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
 OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
 TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
 USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
 DAMAGE.

\vfill
Cover: Ogata Korin. Irises. c1705. Nezu Museum.
}
\end{flushleft}
\thispagestyle{empty}

\newpage
\pagenumbering{roman}
\tableofcontents

\setcounter{section}{0}
\pagenumbering{arabic}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\section*{Changes from Version 1.1}
\begin{enumerate}
\item Addition: Support for eigensolvers.
\item Changes: Specifications of the following functions:
\begin{enumerate}
\item Names of the functions {\tt
      lis\_output\_residual\_history()} and \\
      {\tt lis\_get\_residual\_history()} to 
      {\tt lis\_solver\_output\_rhistory()} and \\
      {\tt lis\_solver\_get\_rhistory()}, respectively. 
\item Origin of the Fortran interfaces {\tt
      lis\_vector\_set\_value()} and \\
      {\tt lis\_vector\_get\_value()} to 1.
\item Origin of the Fortran interface {\tt lis\_vector\_set\_size()} to 1. 
\item Name of the precision flag {\tt -precision} to {\tt -f}.
\end{enumerate}
\item Change: Specification of the function {\tt
      lis\_solve\_kernel()} to return 
      the residual computed by the function {\tt lis\_solve\_execute()}. 
\item Change: Specifications of the integer types:
\begin{enumerate}
\item Replacement: The type of the integer in the C programs with {\tt LIS\_INT},
      which is equivalent to {\tt int} by default.
      If the preprossor macro {\tt \_LONGLONG} is defined, 
      it is replaced with {\tt long long int}.
\item Replacement: The type of the integer in the Fortran programs with {\tt
      LIS\_INTEGER},
      which is equivalent to {\tt integer} by default.
      If the preprossor macro {\tt LONGLONG} is defined, 
      it is replaced with {\tt integer*8}.
\end{enumerate}
\item Change: Names of the matrix storage formats CRS (Compressed Row Storage) 
      and CCS (Compressed Column Storage) to CSR (Compressed Sparse Row) 
      and CSC (Compressed Sparse Column), respectively. 
\item Change: Names of the functions 
{\tt lis\_get\_solvername()}, {\tt lis\_get\_preconname()}, and \\
{\tt lis\_get\_esolvername()} to 
{\tt lis\_solver\_get\_solvername()}, {\tt
      lis\_solver\_get\_preconname()}, and 
{\tt lis\_esolver\_get\_esolvername()}, respectively.
\end{enumerate}

\section*{Changes from Version 1.2}
\begin{enumerate}
\item Change: Name of the file {\tt lis\_config\_win32.h} to {\tt lis\_config\_win.h}.
\item Change: Name of the matrix storage format JDS (Jagged Diagonal Storage) 
      to JAD (Jagged Diagonal). 
\item Change: Names of the functions {\tt
      lis\_fscan\_double()} and {\tt lis\_bswap\_double()} to \\
      {\tt lis\_fscan\_scalar()} and 
      {\tt lis\_bswap\_scalar()}, respectively. 
\end{enumerate}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\section{Introduction}
Lis, a Library of Iterative Solvers for linear systems, is a parallel 
library for solving the linear equations
\[
 Ax = b
\]
and the standard eigenvalue problems
\[
 Ax = \lambda x
\]
with real sparse matrices, which arise in the numerical solution of
partial differential equations, using iterative methods\cite{nishida}.
The solvers available in Lis are listed in Table \ref{tab:solvers} and
\ref{tab:esolvers}, and the preconditioners are listed in Table \ref{tab:precon}.
The supported matrix storage formats are listed in Table \ref{tab:storage}.

\begin{table}[htb]
\begin{minipage}[t]{0.50\textwidth}
\caption{Linear Solvers} 
\vspace*{1mm}
\label{tab:solvers}
\hbox to\hsize{\hfil
\begin{tabular}{l|l}\hline\hline
CG\cite{hestenes,lanczos1} & CR\cite{hestenes} \\ 
BiCG\cite{fletcher}     & BiCR\cite{sogabe09} \\
CGS\cite{sonneveld}     & CRS\cite{abe02} \\
BiCGSTAB\cite{bicgstab} & BiCRSTAB\cite{abe02} \\
GPBiCG\cite{zhang}      & GPBiCR\cite{abe02} \\
BiCGSafe\cite{fujino01} & BiCRSafe\cite{fujino02} \\
BiCGSTAB(l)\cite{bicgstabl} & TFQMR\cite{tfqmr} \\
Jacobi\cite{jacobi}     & Orthomin(m)\cite{orthomin} \\
Gauss-Seidel\cite{gauss,seidel} & GMRES(m)\cite{gmres} \\
SOR\cite{young,frankel} & FGMRES(m)\cite{fgmres} \\
IDR(s)\cite{idrs}       & MINRES\cite{minres} \\
\hline         
\end{tabular}
\hfil}
\end{minipage}
\hspace*{-8mm}
\begin{minipage}[t]{0.30\textwidth}
\caption{Eigensolvers} 
\vspace*{1mm}
\label{tab:esolvers}
\hbox to\hsize{\hfil
\begin{tabular}{l}\hline\hline
Power\cite{power} \\
Inverse\cite{inverse} \\
Approximate Inverse\cite{nishida} \\
Rayleigh Quotient\cite{rayleigh} \\
Subspace\cite{rutishauser} \\
Lanczos\cite{lanczos2} \\
CG\cite{knyazev} \\
CR\cite{suetomi} \\
\hline         
\end{tabular}
\hfil}
\end{minipage}
\newline
\begin{minipage}[t]{0.35\textwidth}
\caption{Preconditioners} 
\vspace*{1mm}
\label{tab:precon}
\hbox to\hsize{\hfil
\begin{tabular}{l}\hline\hline
Jacobi\cite{axelsson} \\
SSOR\cite{axelsson} \\
ILU(k)\cite{gustafsson} \\
ILUT\cite{ilut,ITSOL} \\
Crout ILU\cite{ITSOL,iluc} \\
I+S\cite{kohno01} \\
SA-AMG\cite{fujii01}  \\
Hybrid\cite{abe01} \\
SAINV\cite{bridson01}  \\
Additive Schwarz\cite{chan,dryja} \\
User defined \\
\hline         
\end{tabular}
\hfil}
\end{minipage}
\hspace*{-4mm}
\begin{minipage}[t]{0.35\textwidth}
\caption{Matrix Storage Formats} 
\vspace*{1mm}
\label{tab:storage}
\hbox to\hsize{\hfil
\begin{tabular}{ll}\hline\hline
Compressed Sparse Row & (CSR) \\
Compressed Sparse Column & (CSC) \\
Modified Compressed Sparse Row & (MSR) \\
Diagonal &(DIA) \\
Ellpack-Itpack Generalized Diagonal &(ELL) \\
Jagged Diagonal &(JAD) \\
Block Sparse Row & (BSR) \\
Block Sparse Column &(BSC) \\
Variable Block Row &(VBR) \\
Coordinate & (COO) \\
Dense &	(DNS) \\
\hline         
\end{tabular}
\hfil}
\end{minipage}
\end{table}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\section{Installation}
This section describes the instructions for installing and testing Lis. 
We assume Lis being installed on a server cluster running Linux operating system.

\subsection{System Requirements}
The installation of Lis requires a C compiler. The Fortran interface
requires a Fortran compiler, and the
algebraic multigrid preconditioner requires a Fortran 90 compiler.
For parallel computing environments, an OpenMP\cite{openmp} or 
MPI-1\cite{mpi} library is used\cite{kota1,kota2}.
Both the Harwell-Boeing\cite{duff92} and 
Matrix Market\cite{matrixmarket} formats are supported to import and
export user data. 
Lis has been tested in the environments listed in Table \ref{platforms}
(see also Table \ref{targetoption}). 

\begin{table}[htbp]
\caption{Major Tested Platforms}
\label{platforms}
\begin{center}
{\small
 \begin{tabular}{l|l}
\hline
\multicolumn{1}{c|}{C Compilers} & \multicolumn{1}{c}{OS} \\
\hline
Intel C/C++ Compiler 7.0, 8.0, 9.1, 10.1, 11.1, 12.1 & Linux \\
                                                     & Windows  \\
\hline
IBM XL C/C++ V7.0, 9.0                     & AIX     \\
                                           & Linux   \\
\hline
Sun WorkShop 6, Sun ONE Studio 7,          & Solaris \\
Sun Studio 11, 12                          &         \\
\hline
PGI C++ 6.0, 7.1, 10.5                     & Linux \\
\hline
gcc 3.3, 4.4                               & Linux \\
                                           & Mac OS X \\
                                           & Windows \\
\hline
Microsoft Visual C++ 2008, 2010, 2012      & Windows \\
\hline
\hline
\multicolumn{1}{c|}{Fortran Compilers (Optional)} & \multicolumn{1}{c}{OS} \\
\hline
Intel Fortran Compiler 8.1, 9.1, 10.1, 11.1, 12.1 & Linux \\
                                                  & Windows  \\
\hline
IBM XL Fortran V9.1, 11.1                  & AIX     \\
                                           & Linux   \\
\hline
Sun WorkShop 6, Sun ONE Studio 7,          & Solaris \\
Sun Studio 11, 12                          &         \\
\hline
PGI Fortran 6.0, 7.1, 10.5                 & Linux \\
\hline
g77 3.3                                    & Linux \\
gfortran 4.4                               & Mac OS X \\
                                           & Windows \\
\hline
\end{tabular}
}
\end{center}
\end{table} 

\subsection{Installing on UNIX and Compatible Systems}
\subsubsection{Extracting Archive}
Enter the following command to extract the archive, 
where \verb|($VERSION)| represents the version:\\
 \verb&      > gunzip -c lis-($VERSION).tar.gz | tar xvf - &\\
This creates a directory {\tt lis-(\$VERSION)} together 
with its subfolders as shown in Figure \ref{listargz}.

\begin{figure}[htbp]
\begin{center}
\begin{verbatim}
lis-($VERSION)
 + config
 |  configuration files
 + doc
 |  documents
 + graphics
 |  sample files for graphics
 + include
 |  header files
 + src
 |  source files
 + test
 |  test programs
 + win
    configuration files for Windows systems
\end{verbatim}
\end{center}
\caption{Files contained in {\tt lis-(\$VERSION).tar.gz}}
\label{listargz}
\end{figure}

\subsubsection{Configuring Source Tree}
 In the directory {\tt lis-(\$VERSION)}, run the following command to configure the source tree:
 \begin{itemize}
\item default: \verb&      > ./configure&
\item specify the installation destination: \verb&      > ./configure --prefix=<install-dir>&
\end{itemize}
Table \ref{configoption} shows the major options that can be specified for
the configuration, and 
Table \ref{targetoption} shows the major computing environments that can be specified 
by \verb+TARGET+. 
\begin{table}[htbp]
\caption{Major Configuration Options (see {\tt ./configure --help} for
 the complete list)}
\label{configoption}
\begin{center}
\begin{tabular}{|l|l|}
\hline
\verb+--enable-omp+      & Build with OpenMP library\\ \hline
\verb+--enable-mpi+      & Build with MPI library\\ \hline
\verb+--enable-fortran+  & Build Fortran interface\\ \hline
\verb+--enable-saamg+    & Build SA-AMG preconditioner\\ \hline
\verb+--enable-quad+     & Build with double double (quadruple)
 precision support\\ \hline
\verb+--enable-longdouble+ & Build with long double (quadruple)
 precision support\\ \hline
\verb+--enable-longlong+ & Build with 64bit integer support\\ \hline
\verb+--enable-debug+    & Build with debug mode\\ \hline
\verb+--enable-shared+    & Enable dynamic linking\\ \hline
\verb+--enable-gprof+    & Enable profiling\\ \hline
\verb+--prefix=<install-dir>+    & Specify installation destination\\ \hline
\verb+TARGET=<target>+    & Specify computing environment\\ \hline
\verb+CC=<c_compiler>+    & Specify C compiler\\ \hline
\verb+CFLAGS=<c_flags>+    & Specify options for C compiler\\ \hline
\verb+FC=<fortran_compiler>+    & Specify Fortran compiler\\ \hline
\verb+FCFLAGS=<fc_flags>+    & Specify options for Fortran compiler\\ \hline
\verb+LDFLAGS=<ld_flags>+    & Specify link options\\ \hline
\end{tabular}
\end{center}
\end{table}
\begin{table}[htbp]
\caption{Examples of Targets (see {\tt lis-(\$VERSION)/configure.in} for details)}
\label{targetoption}
\begin{center}
\begin{tabular}{|l|l|}
\hline
\verb+<target>+           & Equivalent options \\ \hline
\verb+cray_xt3_cross+     & \verb+./configure CC=cc FC=ftn CFLAGS="-O3 -B -fastsse -tp k8-64"+ \\
                          & \verb+  FCFLAGS="-O3 -fastsse -tp k8-64 -Mpreprocess" FCLDFLAGS="-Mnomain"+\\
                          & \verb+  ac_cv_sizeof_void_p=8 cross_compiling=yes+\\
                          & \verb+  ax_f77_mangling="lower case, no underscore, extra underscore"+ \\ \hline
\verb+fujitsu_fx10_cross+ & \verb|./configure CC=fccpx FC=frtpx CFLAGS="-Kfast,ocl,preex"| \\
                          & \verb+  FCFLAGS="-Kfast,ocl,preex -Cpp -fs" FCLDFLAGS="-mlcmain=main"+\\
                          & \verb+  ac_cv_sizeof_void_p=8 cross_compiling=yes+\\
                          & \verb+  ax_f77_mangling="lower case, underscore, no extra underscore"+ \\ \hline
\verb+hitachi_sr16k+      & \verb|./configure CC=cc FC=f90 CFLAGS="-Os -noparallel"| \\
                          & \verb+  FCFLAGS="-Oss -noparallel" FCLDFLAGS="-lf90s"+ \\
                          & \verb+  ac_cv_sizeof_void_p=8+ \\
                          & \verb+  ax_f77_mangling="lower case, underscore, no extra underscore" + \\ \hline
\verb+ibm_bgl_cross+      & \verb+./configure CC=blrts_xlc FC=blrts_xlf90+ \\
                          & \verb+  CFLAGS="-O3 -qarch=440d -qtune=440 -qstrict"+ \\
                          & \verb+  FCFLAGS="-O3 -qarch=440d -qtune=440 -qsuffix=cpp=F90"+ \\
                          & \verb+  ac_cv_sizeof_void_p=4 cross_compiling=yes+\\
                          & \verb+  ax_f77_mangling="lower case, no underscore, no extra underscore"+ \\ \hline
\verb+nec_sx9_cross+      & \verb|./configure CC=sxmpic++ FC=sxmpif90 AR=sxar RANLIB=true | \\
                          & \verb+  ac_cv_sizeof_void_p=8 ax_vector_machine=yes cross_compiling=yes+ \\ 
                          & \verb+  ax_f77_mangling="lower case, no underscore, extra underscore"+ \\ \hline
\end{tabular}
\end{center}
\end{table}

\subsubsection{Compiling}
 In the directory {\tt lis-(\$VERSION)}, run the following command to generate
the executable files: \\
 \verb+      > make +\\
To ensure that the library has been built successfully, enter 
\\
 \verb+      > make check+\\
in {\tt lis-(\$VERSION)}. 
This runs a test script using the executable files created 
in {\tt lis-(\$VERSION)/test}, which reads 
the data of the coefficient matrix and the right-hand side vector 
from the file 
{\tt test/testmat.mtx} and solve the linear equation $Ax = b$ 
by the BiCG method. The result on the SGI Altix 3700 is
 shown below. Options {\tt --enable-omp} and {\tt --enable-mpi} can be combined.
\begin{itemsquarebox}[l]{Default}
 \begin{minipage}{10cm}
 \begin{verbatim}
matrix size = 100 x 100 (460 nonzero entries)
initial vector x = 0
precision : double
solver    : BiCG 2
precon    : none
storage   : CSR
lis_solve : normal end

BiCG: number of iterations     = 15 (double = 15, quad = 0)
BiCG: elapsed time             = 5.178690e-03 sec.
BiCG:   preconditioner         = 1.277685e-03 sec. 
BiCG:     matrix creation      = 1.254797e-03 sec.
BiCG:   linear solver          = 3.901005e-03 sec.
BiCG: relative residual 2-norm = 6.327297e-15
 \end{verbatim}
 \end{minipage}
\end{itemsquarebox}
\begin{itemsquarebox}[l]{{\tt --enable-omp}}
 \begin{minipage}{10cm}
 \begin{verbatim}
max number of threads = 32
number of threads = 2
matrix size = 100 x 100 (460 nonzero entries)
initial vector x = 0
precision : double
solver    : BiCG 2
precon    : none
storage   : CSR
lis_solve : normal end

BiCG: number of iterations     = 15 (double = 15, quad = 0)
BiCG: elapsed time             = 8.960009e-03 sec.
BiCG:   preconditioner         = 2.297878e-03 sec. 
BiCG:     matrix creation      = 2.072096e-03 sec.
BiCG:   linear solver          = 6.662130e-03 sec.
BiCG: relative residual 2-norm = 6.221213e-15
 \end{verbatim}
 \end{minipage}
\end{itemsquarebox}
\begin{itemsquarebox}[l]{\tt --enable-mpi}
 \begin{minipage}{10cm}
 \begin{verbatim}
number of processes = 2
matrix size = 100 x 100 (460 nonzero entries)
initial vector x = 0
precision : double
solver    : BiCG 2
precon    : none
storage   : CSR
lis_solve : normal end

BiCG: number of iterations     = 15 (double = 15, quad = 0)
BiCG: elapsed time             = 2.91.4.2e-03 sec.
BiCG:   preconditioner         = 1.560780e-04 sec. 
BiCG:     matrix creation      = 1.459997e-04 sec.
BiCG:   linear solver          = 2.755322e-03 sec.
BiCG: relative residual 2-norm = 6.221213e-15
 \end{verbatim}
 \end{minipage}
\end{itemsquarebox}

\subsubsection{Installing}
In the directory {\tt lis-(\$VERSION)}, enter\\
 \verb+      > make install+\\
which copies the files to the destination directory as follows:

\begin{verbatim}
($INSTALLDIR)
 +bin
 |   +lsolve esolve spmvtest[1-5]
 +include
 |   +lis_config.h lis.h lisf.h
 +lib
 |   +liblis.a
 +share
     +doc/lis examples/lis man
\end{verbatim}

{\tt lis\_config.h} is the header file required to build the library, and 
{\tt lis.h} and {\tt lisf.h} are the header files required 
by the C and Fortran compilers, respectively. 
{\tt liblis.a} is the library file.
To ensure that the library has been installed successfully, enter\\
 \verb+      > make installcheck+\\
in {\tt lis-(\$VERSION)}.
This runs a test script using the executable files installed 
in {\tt examples/lis}. 
{\tt test1} and {\tt etest5} in {\tt examples/lis} are copied in {\tt
(\$INSTALLDIR)/bin} as {\tt lsolve} and {\tt esolve}, respectively.
{\tt examples/lis/spmvtest[1-5]} are also copied in {\tt (\$INSTALLDIR)/bin}.

\subsection{Installing on Windows Systems}
Use an appropriate tool to extract the archive {\tt lis-(\$VERSION).zip}.
To use the Microsoft Build Engine, run the following command in the directory 
{\tt lis-(\$VERSION)\textbackslash win} 
and generate the configuration file {\tt Makefile} (See {\tt configure --help} 
for details): \\
 \verb+      > configure+\\
The default configuration of {\tt Makefile} is defined in {\tt Makefile.in}.
To build the library, run the following command in the directory {\tt
lis-(\$VERSION)\textbackslash win}: \\
 \verb+      > nmake+\\
To ensure that the library has been built successfully, enter\\
 \verb+      > nmake check+\\
The following command copies the library files to {\tt lis-(\$VERSION)\textbackslash lib}
and the executable files to {\tt lis-(\$VERSION)\textbackslash bin}, respectively:\\
 \verb+      > nmake install+\\
To use UNIX compatible environments, follow the instructions in the
previous section. 

\subsection{Testing}
\subsubsection{test1}

\verb+Usage: test1 matrix_filename rhs_setting solution_filename residual_filename [options]+\\

This program inputs the data of the coefficient matrix from {\tt matrix\_filename} and solves the linear equation $Ax=b$ with 
the solver specified by {\tt options}. 
It outputs the solution to 
{\tt solution\_filename} in the extended 
Matrix Market format and 
the residual history to {\tt residual\_filename} in the PLAIN format (see Appendix). 
The Matrix Market and extended Matrix Market formats are supported 
for {\tt matrix\_filename}. 
One of the following values can be specified by {\tt rhs\_setting}:
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[0] Use the right-hand side vector $b$ included in the data file
\item[1] Use $b = (1,\dots,1)^T$
\item[2] Use $b = A \times (1,\dots,1)^T$
\item[\tt rhs\_filename] The filename for the right-hand side vector 
\end{namelist}
The PLAIN and Matrix Market formats are supported for {\tt rhs\_filename}. 
{\tt test1f.F} is the Fortran version of {\tt test1.c}.

\subsubsection{test2}

\verb+Usage: test2 m n matrix_type solution_filename residual_filename [options]+\\

This program solves the linear equation $Ax = b$, where the coefficient 
matrix $A$ of size $mn$ is derived from a discretized two dimensional 
Poisson's equation using the five
point central difference scheme, with the coefficient matrix 
in the storage format specified
by \verb|matrix_type| and the solver specified by {\tt options}. 
It outputs the solution to {\tt solution\_filename} in the extended 
Matrix Market format and 
the residual history to {\tt residual\_filename} in the PLAIN format. 
The right-hand side vector is set such that the values of the elements
of the solution are $1$. 
The values {\tt m} and  {\tt n} represent the numbers of grid points
in each dimension. 

\subsubsection{test3}

\verb+Usage: test3 l m n matrix_type solution_filename residual_filename [options]+\\

This program solves the linear equation $Ax = b$, where the coefficient 
matrix $A$ of size $lmn$ is derived from a discretized three dimensional 
Poisson's equation using the seven
point central difference scheme, with the coefficient matrix 
in the storage format specified
by \verb|matrix_type| and the solver specified by {\tt options}. 
It outputs the solution to {\tt solution\_filename} in the extended 
Matrix Market format and 
the residual history to {\tt residual\_filename} in the PLAIN format. 
The right-hand side vector is set such that the values of the elements of the solution are $1$. 
The values {\tt l}, {\tt m} and {\tt n} represent the numbers of grid
points in each dimension. 

\subsubsection{test4}
This program solves the linear equation $Ax = b$ with a specified 
solver and a preconditioner, where $A$ is a tridiagonal matrix
\[
\left(
\begin{array}{ccccc}
2 & -1 &   &  &   \\
-1 & 2 & -1 &  &   \\
  & \ddots  & \ddots  & \ddots  &   \\
  &   & -1 & 2 & -1 \\
  &   &   & -1 & 2 \\
\end{array}
\right)
\]
of size $12$.
The right-hand side vector $b$ is set such that  
the values of the elements of the solution $x$ are $1$. 
{\tt test4f.F} is the Fortran version of {\tt test4.c}.

\subsubsection{test5}

\verb+Usage: test5 n gamma [options]+\\

This program solves a linear equation $Ax =b$, where $A$ is a Toeplitz matrix
\[
\left(
\begin{array}{cccccc}
2 & 1 &   &  &  & \\
0 & 2 & 1 &  &  & \\
\gamma & 0& 2 & 1 &  & \\
 & \ddots & \ddots & \ddots & \ddots & \\
 &  &   \gamma &0 &       2   & 1 \\
 &  &  &   \gamma & 0& 2 \\
\end{array}
\right)
\]
of size $n$, with the solver specified by {\tt options}. 
Note that the right-hand vector is set such that the values of the elements 
of the solution are $1$. 

\subsubsection{etest1}

\verb+Usage: etest1 matrix_filename solution_filename residual_filename [options]+\\

This program inputs the matrix data from {\tt matrix\_filename} and
solves the eigenvalue problem $Ax=\lambda x$ with 
the solver specified by {\tt options}. 
It outputs the associated eigenvector to 
{\tt solution\_filename} in the extended Matrix Market format and 
the residual history to {\tt residual\_filename} in the PLAIN format.
The Matrix Market format is supported for {\tt matrix\_filename}. 
{\tt etest1f.F} is the Fortran version of {\tt etest1.c}.

\subsubsection{etest2}

\verb+Usage: etest2 m n matrix_type solution_filename residual_filename [options]+\\

This program solves the eigenvalue problem $Ax = \lambda x$, where the 
coefficient matrix $A$ of size $mn$ is derived from a discretized two dimensional Helmholtz's equation using the five
point central difference scheme, with the coefficient matrix in the storage format specified
by \verb|matrix_type| and the solver specified by {\tt options}. 
It outputs the associated eigenvector to {\tt solution\_filename} in the
extended Matrix Market format and 
the residual history to {\tt residual\_filename} in the PLAIN format. 
The values {\tt m} and {\tt m} represent the numbers of grid points
in each dimension. 

\subsubsection{etest3}

\verb+Usage: etest3 l m n matrix_type solution_filename residual_filename [options]+\\

This program solves the eigenvalue problem $Ax = \lambda x$, where the 
coefficient matrix $A$ of size $lmn$ is derived from a discretized three dimensional Helmholtz's equation using the seven
point central difference scheme, with the coefficient matrix in the storage format specified
by \verb|matrix_type| and the solver specified by {\tt options}. 
It outputs the associated eigenvector to {\tt solution\_filename} in the
extended Matrix Market format and
the residual history to {\tt residual\_filename} in the PLAIN format. 
The values {\tt l}, {\tt m} and {\tt n} represent the numbers of grid
points in each dimension. 

\subsubsection{etest4}

\verb+Usage: etest4 n [options]+\\

This program solves the eigenvalue problem $Ax = \lambda x$ with a specified 
solver, where $A$ is a tridiagonal matrix
\[
A = 
\left(
\begin{array}{ccccc}
2 & -1 &   &  &   \\
-1 & 2 & -1 &  &   \\
  & \ddots  & \ddots  & \ddots  &   \\
  &   & -1 & 2 & -1 \\
  &   &   & -1 & 2 \\
\end{array}
\right)
\]
of size $n \times n$.
{\tt etest4f.F} is the Fortran version of {\tt etest4.c}.

\subsubsection{etest5}

\verb+Usage: etest5 matrix_filename eigenvalue_filename eigenvector_filename [options] +\\

This program inputs the matrix data from {\tt matrix\_filename} and
solves the eigenvalue problem $Ax=\lambda x$ with
the solver specified by {\tt options}. 
It outputs the eigenvalues specified by {\tt options} to 
{\tt eigenvalue\_filename} and the associated eigenvectors to 
{\tt eigenvector\_filename} in the extended Matrix Market format. The Matrix Market format is supported for {\tt matrix\_filename}. 

\subsubsection{etest6}

\verb+Usage: etest6 l m n matrix_type eigenvalue_filename eigenvector_filename [options] +\\

This program solves the eigenvalue problem $Ax = \lambda x$, where the 
coefficient matrix $A$ of size $lmn$ is derived from a discretized three dimensional Helmholtz's equation using the seven
point central difference scheme, with the coefficient matrix in the storage format specified
by \verb|matrix_type| and the solver specified by {\tt options}. 
It outputs the eigenvalues specified by {\tt options} to 
{\tt eigenvalue\_filename} and the associated eigenvectors to 
{\tt eigenvector\_filename} in the extended Matrix Market format.
The values {\tt l}, {\tt m} and {\tt n} represent the numbers of grid
points in each dimension. 

\subsubsection{spmvtest1}

\verb+Usage: spmvtest1 n iter [matrix_type]+\\

This program computes the matrix-vector multiply of a discretized one dimensional Laplacian
\[
\left(
\begin{array}{ccccc}
2 & -1 &   &  &   \\
-1 & 2 & -1 &  &   \\
  & \ddots  & \ddots  & \ddots  &   \\
  &   & -1 & 2 & -1 \\
  &   &   & -1 & 2 \\
\end{array}
\right)
\]
of size $n$ using the three point central difference scheme
and a vector $(1,\dots,1)^T$.
The FLOPS performance is measured as the average of {\tt iter}
iterations.
If necessary, one of the following values can be specified by {\tt matrix\_type}:
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[0] Measure the performance for the available matrix storage formats
\item[1-11] The number of the matrix storage format
\end{namelist}

\subsubsection{spmvtest2}

\verb+Usage: spmvtest2 m n iter [matrix_type]+\\

This program computes the matrix-vector multiply of a 
discretized two dimensional Laplacian of size $mn$ using 
the five point central difference scheme 
and a vector $(1,\dots,1)^T$. 
The FLOPS performance is measured as the average of {\tt iter}
iterations.
If necessary, one of the following values can be specified by {\tt matrix\_type}:
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[0] Measure the performance for the available matrix storage formats
\item[1-11] The number of the matrix storage format
\end{namelist}
The values {\tt m} and {\tt n} represent the numbers of grid points 
in each dimension. 

\subsubsection{spmvtest3}

\verb+Usage: spmvtest3 l m n iter [matrix_type]+\\

This program computes the matrix-vector multiply of a 
discretized three dimensional Laplacian of size $lmn$ using 
the seven point central difference scheme and a vector $(1,\dots,1)^T$. 
The values {\tt l}, {\tt m} and {\tt n} represent the numbers of grid
points in each dimension. 
The FLOPS performance is measured as the average of {\tt iter}
iterations.
If necessary, one of the following values can be specified by {\tt matrix\_type}:
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[0] Measure the performance for the available matrix storage formats
\item[1-11] The number of the matrix storage format
\end{namelist}

\subsubsection{spmvtest4}

\verb+Usage: spmvtest4 matrix_filename_list iter [block]+\\

This program inputs the matrix data from the files listed in {\tt matrix\_filename\_list}, 
and computes the multiplies of matrices in available matrix 
storage formats and a vector $(1,\dots,1)^T$. 
The FLOPS performance is measured as the average of {\tt iter}
iterations. 
The Matrix Market format is supported for {\tt matrix\_filename\_list}. 
If necessary, the block size of the BSR and BSC formats can be specified
by {\tt block}.

\subsubsection{spmvtest5}

\verb+Usage: spmvtest5 matrix_filename matrix_type iter [block]+\\

This program inputs the matrix data from {\tt matrix\_filename} 
and compute the multiply of the matrix 
with \verb|matrix_type| and a vector $(1,\dots,1)^T$. 
The FLOPS performance is measured as the average of {\tt iter} iterations. 
The Matrix Market format is supported for {\tt matrix\_filename}. 
If necessary, the block size of the BSR and BSC formats can be specified by {\tt block}.

\subsection{Limitations}
The current version has the following limitations:
\begin{itemize}
\item Preconditioners
\begin{itemize}
\item If a preconditioner other than the Jacobi or SSOR is selected 
      and matrix $A$ is not in the CSR format, a new matrix is created 
      in the CSR format for preconditioning.
\item The SA-AMG preconditioner does not support the BiCG method. 
\item The SA-AMG preconditioner does not support the multithreaded environment. 
\item The assembly of the matrices in the SAINV preconditioner is not parallelized. 

\end{itemize}

\item Matrix storage formats
\begin{itemize}
\item The VBR format does not support the multiprocessing environment.
\item The SA-AMG preconditioner supports only the CSR format. 
\item In the multiprocessing environment, the CSR is the only accepted
      format for user defined arrays.
\end{itemize}

\item Double double (quadruple) precision operations (see Section \ref{sec:quadruple})
\begin{itemize}
\item The Jacobi, Gauss-Seidel, SOR, and IDR(s) methods do not support
      the double double precision operations.
\item The CG and CR methods for the eigenvalue problems do not support the double double precision operations.
\item The Jacobi, Gauss-Seidel and SOR methods in the hybrid preconditioner do not support the double double precision operations.
\item The I+S and SA-AMG preconditioners do not support the double double precision operations.
\end{itemize}

\item Long double (quadruple) precision operations
\begin{itemize}
\item The SA-AMG preconditioner does not support the long double precision operations.
\end{itemize}

\end{itemize}
\vspace*{5mm}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\section{Basic Operations}
This section describes how to use the library. 
A program requires the following statements:
\begin{itemize}
\item Initialization
\item Matrix creation
\item Vector creation
\item Solver creation
\item Value assignment for matrices and vectors
\item Solver assignment
\item Solver execution
\item Finalization
\end{itemize}
In addition, it must include one of the following compiler directives: 
\begin{itemize}
\item \verb+C       #include "lis.h"+
\item \verb+Fortran #include "lisf.h"+
\end{itemize}
When Lis is installed in \verb|($INSTALLDIR)|, {\tt lis.h} and {\tt lisf.h}
are located in \verb|($INSTALLDIR)/include|.

\subsection{Initializing and Finalizing}
The functions for initializing and finalizing the execution environment 
must be called at the top and bottom of the program, respectively, as follows:
\begin{itemsquarebox}[l]{C}
\small
\begin{verbatim}
 1: #include "lis.h"
 2: LIS_INT main(LIS_INT argc, char* argv[])
 3: {
 4:     lis_initialize(&argc, &argv);
 5:     ...
 6:     lis_finalize();
 7: }
\end{verbatim}
\end{itemsquarebox}
\begin{itemsquarebox}[l]{Fortran}
\small
\begin{verbatim}
 1: #include "lisf.h"
 2:      call lis_initialize(ierr) 
 3:     ...
 4:      call lis_finalize(ierr)
\end{verbatim}
\end{itemsquarebox}
\\ \\
\noindent
{\bf Initializing}

For initializing, the following functions are used:
\begin{itemize}
\item \verb+C       LIS_INT lis_initialize(LIS_INT* argc, char** argv[])+
\item \verb+Fortran subroutine lis_initialize(LIS_INTEGER ierr)+
\end{itemize}
This function initializes the MPI execution environment, 
and specifies the options on the command line.

The default type of the integer in the C programs is {\tt LIS\_INT}, which
is equivalent to {\tt int}.  If the preprossor macro {\tt \_LONGLONG}
is defined, it is replaced with {\tt long long int}.
The default type of the integer in the Fortran programs is 
{\tt LIS\_INTEGER}, which is equivalent to {\tt integer}.  If the
preprocessor macro {\tt LONGLONG} is defined, it is replaced with 
{\tt integer*8}.
\\ \\
\noindent
{\bf Finalizing}

For finalizing, the following functions are used: 
\begin{itemize}
\item \verb+C       LIS_INT lis_finalize()+
\item \verb+Fortran subroutine lis_finalize(LIS_INTEGER ierr)+
\end{itemize}

\subsection{Operating Vectors}
Assume that the size of vector $v$ is $global\_n$, and the size 
of each partial vector stored on $nprocs$ processing elements is $local\_n$. 
If $global\_n$ is divisible, 
then $local\_n$ is equal to $global\_n$ $/$ $nprocs$. 
For example, when vector $v$ is stored on two processing elements, 
as shown in Equation (\ref{eq:vecv}), $global\_n$ and $local\_n$ 
are $4$ and $2$, respectively.
\begin{equation}
v = 
\left(
\begin{array}{c}
0 \\
1 \\ \hline
2 \\
3  
\end{array}
\right)
\begin{array}{l}
\mbox{PE0} \\
    \\
\mbox{PE1} \\
   \\ 
\end{array}
\label{eq:vecv}
\end{equation}

In the case of creating vector $v$ in Equation (\ref{eq:vecv}), 
vector $v$ itself is created for the serial and multithreaded environments, while 
the partial vectors are created and stored on a given number of processing elements
for the multiprocessing environment.
 
Programs to create vector $v$ are as follows, 
where the number of processing elements for the multiprocessing environment is assumed to be two:
\begin{itemsquarebox}[l]{C (for serial and multithreaded environments)}
\small
\begin{verbatim}
 1: LIS_INT       i,n;
 2: LIS_VECTOR    v;
 3: n = 4;
 4: lis_vector_create(0,&v);
 5: lis_vector_set_size(v,0,n);              /* or lis_vector_set_size(v,n,0); */ 
 6:
 7: for(i=0;i<n;i++)
 8: {
 9:     lis_vector_set_value(LIS_INS_VALUE,i,(double)i,v);
10:  }
\end{verbatim}
\end{itemsquarebox}
\begin{itemsquarebox}[l]{C (for multiprocessing environment)}
\small
\begin{verbatim}
 1: LIS_INT       i,n,is,ie;                 /* or LIS_INT  i,ln,is,ie; */
 2: LIS_VECTOR    v;
 3: n = 4;                                   /* ln = 2; */
 4: lis_vector_create(MPI_COMM_WORLD,&v);
 5: lis_vector_set_size(v,0,n);              /* lis_vector_set_size(v,ln,0); */
 6: lis_vector_get_range(v,&is,&ie);
 7: for(i=is;i<ie;i++)
 8: {
 9:     lis_vector_set_value(LIS_INS_VALUE,i,(double)i,v);
10:  }
\end{verbatim}
\end{itemsquarebox}
\begin{itemsquarebox}[l]{Fortran (for serial and multithreaded environments)}
\small
\begin{verbatim}
 1: LIS_INTEGER   i,n
 2: LIS_VECTOR    v
 3: n = 4
 4: call lis_vector_create(0,v,ierr)
 5: call lis_vector_set_size(v,0,n,ierr)  
 6:
 7: do i=1,n
 9:     call lis_vector_set_value(LIS_INS_VALUE,i,DBLE(i),v,ierr)
10: enddo
\end{verbatim}
\end{itemsquarebox}
\begin{itemsquarebox}[l]{Fortran (for multiprocessing environment)}
\small
\begin{verbatim}
 1: LIS_INTEGER   i,n,is,ie                 
 2: LIS_VECTOR    v
 3: n = 4                                   
 4: call lis_vector_create(MPI_COMM_WORLD,v,ierr)
 5: call lis_vector_set_size(v,0,n,ierr)              
 6: call lis_vector_get_range(v,is,ie,ierr)
 7: do i=is,ie-1
 8:     call lis_vector_set_value(LIS_INS_VALUE,i,DBLE(i),v,ierr);
 9: enddo
\end{verbatim}
\end{itemsquarebox}
\\ \\
\noindent
{\bf Declaring Variables}

As the second line shows, the declaration is stated as follows:\\ \\
\verb|    LIS_VECTOR    v;|\\
\\ 
\noindent
{\bf Creating Vectors}

To create vector $v$, the following functions are used: 
\begin{itemize}
\item \verb|C       LIS_INT lis_vector_create(LIS_Comm comm, LIS_VECTOR *v)|
\item \verb|Fortran subroutine lis_vector_create(LIS_Comm comm, LIS_VECTOr v, LIS_INTEGER ierr)|
\end{itemize}
For the example program above, {\tt comm} must be replaced with the MPI communicator. 
For the serial and multithreaded environments, the value of {\tt comm} is ignored.
\\ \\
\noindent
{\bf Assigning Sizes}

To assign a size to vector $v$, the following functions are used: 
\begin{itemize}
\item \verb|C       LIS_INT lis_vector_set_size(LIS_VECTOR v, LIS_INT local_n, LIS_INT global_n)|
\item \verb|Fortran subroutine lis_vector_set_size(LIS_VECTOR v, LIS_INTEGER local_n,|\\
      \verb|         LIS_INTEGER global_n, LIS_INTEGER ierr)| 
\end{itemize}
Either $local\_n$ or $global\_n$ must be provided.

For the serial and multithreaded environments, $local\_n$ 
is equal to $global\_n$. 
Therefore, both \\
\verb|lis_vector_set_size(v,n,0)| and \verb|lis_vector_set_size(v,0,n)| create a vector of size $n$. 

For the multiprocessing environment, \verb|lis_vector_set_size(v,n,0)| 
creates a partial vector of size $n$ on each processing element. On the other hand, 
\verb|lis_vector_set_size(v,0,n)| creates a partial vector of size 
$m_p$ on processing element $p$. The values of $m_p$ are determined by the library. 
\\ \\
\noindent
{\bf Assigning Values}

To assign a value to the $i$-th element of vector $v$, the following functions are used:
\begin{itemize}
\item \verb|C       LIS_INT lis_vector_set_value(LIS_INT flag, LIS_INT i, LIS_SCALAR value,|\\
      \verb|         LIS_VECTOR v)|
\item \verb|Fortran subroutine lis_vector_set_value(LIS_INT flag, LIS_INT i, LIS_SCALAR value,|\\
      \verb|         LIS_VECTOR v, LIS_INTEGER ierr)|
\end{itemize}
For the multiprocessing environment, the $i$-th row of the global vector must be
specified. 
Either
\begin{description}
\item[\tt LIS\_INS\_VALUE]: $v[i] = value$, or
\item[\tt LIS\_ADD\_VALUE]: $v[i] = v[i] + value$
\end{description}
must be provided for \verb+flag+.
\\ \\
\noindent
{\bf Duplicating Vectors}

To create a vector that has the same information as the existing vector, 
the following functions are used:
\begin{itemize}
\item \verb|C       LIS_INT lis_vector_duplicate(LIS_VECTOR vin, LIS_VECTOR *vout)|
\item \verb|Fortran subroutine lis_vector_duplicate(LIS_VECTOR vin, LIS_VECTOR vout,|\\
      \verb|         LIS_INTEGER ierr)|
\end{itemize}
This function does not copy the values of the vector. 
To copy the values as well, the following functions must be called after the above functions:
\begin{itemize}
\item \verb|C       LIS_INT lis_vector_copy(LIS_VECTOR vsrc, LIS_VECTOR vdst)|
\item \verb|Fortran subroutine lis_vector_copy(LIS_VECTOR vsrc, LIS_VECTOR vdst,|\\
      \verb|        LIS_INTEGER ierr)| 
\end{itemize}
\noindent
{\bf Destroying Vectors}

To destroy the vector, the following functions are used:
\begin{itemize}
\item \verb|C       LIS_INT lis_vector_destroy(LIS_VECTOR v)|
\item \verb|Fortran subroutine lis_vector_destroy(LIS_VECTOR v,|\\
      \verb|        LIS_INTEGER ierr)|
\end{itemize}

\subsection{Operating Matrices}
Assume that the size of matrix $A$ is 
$global\_n$ $\times$ $global\_n$, and that the size of each row block
of matrix $A$ stored on $nprocs$ processing elements is 
$local\_n$ $\times$ $global\_n$. 
If $global\_n$ is divisible, 
then $local\_n$ is equal to $global\_n$ $/$ $nprocs$. 
For example, when the row block of matrix $A$ is stored on two
processing elements, 
as shown in Equation (\ref{eq:mat}), $global\_n$ and $local\_n$ 
are $4$ and $2$, respectively.

\begin{equation}
\label{eq:mat}
A = 
\left(
\begin{array}{cccc}
2 & 1 &   &    \\
1 & 2 & 1 &    \\ \hline
  & 1 & 2 & 1 \\
  &   & 1 & 2 
\end{array}
\right)
\begin{array}{l}
\mbox{PE0} \\
    \\
\mbox{PE1} \\
   \\ 
\end{array}
\end{equation}

A matrix in a specific storage format can be created in one of the
following three ways:\\ \\
\noindent
{\bf Method 1: Define Arrays in a Specific Storage Format with Library Functions}
\\ \\
\indent
For creating matrix $A$ in Equation (\ref{eq:mat}) in the CSR format, 
matrix $A$ itself is created for the serial and multithreaded environments,
while partial matrices are created and 
stored on the given number of processing elements 
for the multiprocessing environment.

Programs to create matrix $A$ in the CSR format are as follows, 
where the number of processing elements for the multiprocessing environment is assumed to be two:
\begin{itemsquarebox}[l]{C (for serial and multithreaded environments)}
\small
\begin{verbatim}
 1: LIS_INT       i,n;
 2: LIS_MATRIX    A;
 3: n = 4;
 4: lis_matrix_create(0,&A);
 5: lis_matrix_set_size(A,0,n);              /* or lis_matrix_set_size(A,n,0); */ 
 6: for(i=0;i<n;i++) {
 7:     if( i>0   ) lis_matrix_set_value(LIS_INS_VALUE,i,i-1,1.0,A);
 8:     if( i<n-1 ) lis_matrix_set_value(LIS_INS_VALUE,i,i+1,1.0,A);
 9:     lis_matrix_set_value(LIS_INS_VALUE,i,i,2.0,A);
10:  }
11:  lis_matrix_set_type(A,LIS_MATRIX_CSR);
12:  lis_matrix_assemble(A);
\end{verbatim}
\end{itemsquarebox}
\begin{itemsquarebox}[l]{C (for multiprocessing environment)}
\small
\begin{verbatim}
 1: LIS_INT       i,n,gn,is,ie;                 
 2: LIS_MATRIX    A;
 3: gn = 4;                                  /* or n=2 */
 4: lis_matrix_create(MPI_COMM_WORLD,&A);
 5: lis_matrix_set_size(A,0,gn);             /* lis_matrix_set_size(A,n,0); */
 6: lis_matrix_get_size(A,&n,&gn);
 7: lis_matrix_get_range(A,&is,&ie);
 8: for(i=is;i<ie;i++) {
 9:     if( i>0    ) lis_matrix_set_value(LIS_INS_VALUE,i,i-1,1.0,A);
10:     if( i<gn-1 ) lis_matrix_set_value(LIS_INS_VALUE,i,i+1,1.0,A);
11:     lis_matrix_set_value(LIS_INS_VALUE,i,i,2.0,A);
12:  }
13:  lis_matrix_set_type(A,LIS_MATRIX_CSR);
14:  lis_matrix_assemble(A);
\end{verbatim}
\end{itemsquarebox}
\begin{itemsquarebox}[l]{Fortran (for serial and multithreaded environments)}
\small
\begin{verbatim}
 1: LIS_INTEGER   i,n
 2: LIS_MATRIX    A
 3: n = 4
 4: call lis_matrix_create(0,A,ierr)
 5: call lis_matrix_set_size(A,0,n,ierr)
 6: do i=1,n
 7:     if( i>1 ) call lis_matrix_set_value(LIS_INS_VALUE,i,i-1,1.0d0,A,ierr)
 8:     if( i<n ) call lis_matrix_set_value(LIS_INS_VALUE,i,i+1,1.0d0,A,ierr)
 9:     call lis_matrix_set_value(LIS_INS_VALUE,i,i,2.0d0,A,ierr)
10:  enddo
11:  call lis_matrix_set_type(A,LIS_MATRIX_CSR,ierr)
12:  call lis_matrix_assemble(A,ierr)
\end{verbatim}
\end{itemsquarebox}
\begin{itemsquarebox}[l]{Fortran (for multiprocessing environment)}
\small
\begin{verbatim}
 1: LIS_INTEGER   i,n,gn,is,ie                 
 2: LIS_MATRIX    A
 3: gn = 4
 4: call lis_matrix_create(MPI_COMM_WORLD,A,ierr)
 5: call lis_matrix_set_size(A,0,gn,ierr)
 6: call lis_matrix_get_size(A,n,gn,ierr)
 7: call lis_matrix_get_range(A,is,ie,ierr)
 8: do i=is,ie-1
 9:     if( i>1  ) call lis_matrix_set_value(LIS_INS_VALUE,i,i-1,1.0d0,A,ierr)
10:     if( i<gn ) call lis_matrix_set_value(LIS_INS_VALUE,i,i+1,1.0d0,A,ierr)
11:     call lis_matrix_set_value(LIS_INS_VALUE,i,i,2.0d0,A,ierr)
12:  enddo
13:  call lis_matrix_set_type(A,LIS_MATRIX_CSR,ierr)
14:  call lis_matrix_assemble(A,ierr)
\end{verbatim}
\end{itemsquarebox}
\\ \\
\noindent
{\bf Declaring Variables}

As the second line shows, the declaration is stated as:\\ \\
\verb|    LIS_MATRIX    A;|\\
\\ \\
\noindent
{\bf Creating Matrices}

To create matrix $A$, the following functions are used: 
\begin{itemize}
\item \verb|C       LIS_INT lis_matrix_create(LIS_Comm comm, LIS_MATRIX *A)|
\item \verb|Fortran subroutine lis_matrix_create(LIS_Comm comm, LIS_MATRIX A, LIS_INTEGER ierr)|
\end{itemize}
{\tt comm} must be replaced with the MPI communicator. 
For the serial and multithreaded environments, the value of {\tt comm} is ignored.
\\ \\
\noindent
{\bf Assigning Sizes}

To assign a size to matrix $A$, the following functions are used: 
\begin{itemize}
\item \verb|C       LIS_INT lis_matrix_set_size(LIS_MATRIX A, LIS_INT local_n, LIS_INT global_n)|
\item \verb|Fortran subroutine lis_matrix_set_size(LIS_MATRIX A, LIS_INTEGER local_n,|\\
      \verb|         LIS_INTEGER global_n, LIS_INTEGER ierr)|
\end{itemize}
Either $local\_n$ or $global\_n$ must be provided. 

For the serial and multithreaded environments, $local\_n$ is
equal to $global\_n$. 
Therefore, both \\
\verb|lis_matrix_set_size(A,n,0)| and \verb|lis_matrix_set_size(A,0,n)|
create a matrix of size $n \times n$.

For the multiprocessing environment, \verb|lis_matrix_set_size(A,n,0)| creates
a partial matrix of size $n \times N$ on each processing element, 
where $N$ is the total sum of $n$. 
On the other hand, \verb|lis_matrix_set_size(A,0,n)| creates 
a partial matrix of size $m_p \times n$ on processing element $p$.
The values of $m_p$ are determined by the library.
\\ \\
\noindent
{\bf Assigning Values}

To assign a value to the element at the $i$-th row and the $j$-th column of
matrix $A$, 
the following functions are used: 
\begin{itemize}
\item \verb|C       LIS_INT lis_matrix_set_value(LIS_INT flag, LIS_INT i, LIS_INT j,|\\
      \verb|         LIS_SCALAR value, LIS_MATRIX A)|
\item \verb|Fortran subroutine lis_matrix_set_value(LIS_INTEGER flag, LIS_INTEGER i,|\\
      \verb|         LIS_INTEGER j, LIS_SCALAR value, LIS_MATRIX A, LIS_INTEGER ierr)|
\end{itemize}
For the multiprocessing environment, the $i$-th row and the $j$-th column of the global matrix 
must be specified. 
Either
\begin{description}
\item[\tt LIS\_INS\_VALUE]: $A(i,j) = value$, or
\item[\tt LIS\_ADD\_VALUE]: $A(i,j) = A(i,j) + value$
\end{description}
must be provided for the parameter \verb+flag+.
\\ \\
\noindent
{\bf Assigning Storage Formats}

To assign a storage format to matrix $A$, 
the following functions are used: 
\begin{itemize}
\item \verb|C       LIS_INT lis_matrix_set_type(LIS_MATRIX A, LIS_INT matrix_type)|
\item \verb|Fortran subroutine lis_matrix_set_type(LIS_MATRIX A, LIS_INT matrix_type,|\\
      \verb|         LIS_INTEGER ierr)|
\end{itemize}
where \verb+matrix_type+ is \verb+LIS_MATRIX_CSR+ when the matrix is created. 
The following storage formats are supported:
\\ \\
\begin{minipage}[t]{\textwidth}
\begin{center}
\begin{tabular}{lll}\hline\hline
Storage format  & & \verb|matrix_type| \\ \hline
Compressed Sparse Row & (CSR) & \verb={LIS_MATRIX_CSR|1}= \\
Compressed Sparse Column & (CSC) & \verb={LIS_MATRIX_CSC|2}= \\
Modified Compressed Sparse Row & (MSR) & \verb={LIS_MATRIX_MSR|3}= \\
Diagonal &(DIA) & \verb={LIS_MATRIX_DIA|4}= \\
Ellpack-Itpack Generalized Diagonal &(ELL) & \verb={LIS_MATRIX_ELL|5}= \\
Jagged Diagonal &(JAD) & \verb={LIS_MATRIX_JAD|6}= \\
Block Sparse Row & (BSR) & \verb={LIS_MATRIX_BSR|7}= \\
Block Sparse Column &(BSC) & \verb={LIS_MATRIX_BSC|8}= \\
Variable Block Row &(VBR) & \verb={LIS_MATRIX_VBR|9}= \\
Coordinate & (COO) & \verb={LIS_MATRIX_COO|10}= \\
Dense &	(DNS) & \verb={LIS_MATRIX_DNS|11}= \\
\hline         
\end{tabular}
\end{center}
\end{minipage}
\\ \\ \\
\noindent
{\bf Assembling Matrices}

After assigning values and storage formats, the following functions must be called:
\begin{itemize}
\item \verb|C       LIS_INT lis_matrix_assemble(LIS_MATRIX A)|
\item \verb|Fortran subroutine lis_matrix_assemble(LIS_MATRIX A, LIS_INTEGER ierr)|
\end{itemize}
\verb|lis_matrix_assemble| assembles $A$ into the storage format specified by \verb|lis_matrix_set_type|.
\\ \\ 
\noindent
{\bf Destroying Matrices}

To destroy the matrix, the following functions are used:
\begin{itemize}
\item \verb|C       LIS_INT lis_matrix_destroy(LIS_MATRIX A)|
\item \verb|Fortran subroutine lis_matrix_destroy(LIS_MATRIX A, LIS_INTEGER ierr)|
\end{itemize}
\vspace*{5mm}

\noindent
{\bf Method 2: Define Arrays in a Specific Storage Format Directly}\\
\\ 
\indent
For creating matrix $A$ in Equation (\ref{eq:mat}) in the
CSR format, 
matrix $A$ itself is created for the serial and multithreaded environments,
while the partial matrices are created and 
stored on the given number of processing elements 
for the multiprocessing environment.

Programs to create matrix $A$ in the CSR format are as follows,
where the number of processing elements for the multiprocessing environment is assumed to be two: 
\begin{itemsquarebox}[l]{C (for serial and multithreaded environments)}
\small
\begin{verbatim}
 1: LIS_INT       i,k,n,nnz;
 2: LIS_INT       *ptr,*index;
 3: LIS_SCALAR    *value;
 4: LIS_MATRIX    A;
 5: n = 4; nnz = 10; k = 0;
 6: lis_matrix_malloc_csr(n,nnz,&ptr,&index,&value);
 7: lis_matrix_create(0,&A);
 8: lis_matrix_set_size(A,0,n);              /* or lis_matrix_set_size(A,n,0); */ 
 9: 
10: for(i=0;i<n;i++)
11: {
12:     if( i>0   ) {index[k] = i-1; value[k] = 1; k++;}
13:     index[k] = i; value[k] = 2; k++;
14:     if( i<n-1 ) {index[k] = i+1; value[k] = 1; k++;}
15:     ptr[i+1] = k;
16:  }
17:  ptr[0] = 0;
18:  lis_matrix_set_csr(nnz,ptr,index,value,A);
19:  lis_matrix_assemble(A); 


\end{verbatim}
\end{itemsquarebox}
\begin{itemsquarebox}[l]{C (for multiprocessing environment)}
\small
\begin{verbatim}
 1: LIS_INT       i,k,n,nnz,is,ie;
 2: LIS_INT       *ptr,*index;
 3: LIS_SCALAR    *value;
 4: LIS_MATRIX    A;
 5: n = 2; nnz = 5; k = 0;
 6: lis_matrix_malloc_csr(n,nnz,&ptr,&index,&value);
 7: lis_matrix_create(MPI_COMM_WORLD,&A);
 8: lis_matrix_set_size(A,n,0);
 9: lis_matrix_get_range(A,&is,&ie);
10: for(i=is;i<ie;i++)
11: {
12:     if( i>0   ) {index[k] = i-1; value[k] = 1; k++;}
13:     index[k] = i; value[k] = 2; k++;
14:     if( i<n-1 ) {index[k] = i+1; value[k] = 1; k++;}
15:     ptr[i-is+1] = k;
16:  }
17:  ptr[0] = 0;
18:  lis_matrix_set_csr(nnz,ptr,index,value,A);
19:  lis_matrix_assemble(A); 
\end{verbatim}
\end{itemsquarebox}
\\ \\
\noindent
{\bf Associating Arrays}

To associate the arrays in the CSR format with matrix $A$, the following functions are used:
\begin{itemize}
\item \verb|C       LIS_INT lis_matrix_set_csr(LIS_INT nnz, LIS_INT row[], LIS_INT index[],|\\
      \verb|         LIS_SCALAR value[], LIS_MATRIX A)|
\item \verb|Fortran subroutine lis_matrix_set_csr(LIS_INTEGER nnz, LIS_INTEGER row(),|\\
      \verb|         LIS_INTEGER index(), LIS_SCALAR value(), LIS_MATRIX A, LIS_INTEGER ierr)|
\end{itemize}
\vspace*{5mm}

\noindent
{\bf Method 3: Read Matrix and Vector Data from External Files}\\
\\ \indent
Programs to read matrix $A$ in Equation (\ref{eq:mat}) in the CSR format and 
vector $b$ in Equation (\ref{eq:vecv}) from an external file are as follows: 
\\
\begin{itemsquarebox}[l]{C (for serial, multithreaded and multiprocessing environments)}
\small
\begin{verbatim}
 1: LIS_MATRIX    A;
 2: LIS_VECTOR    b,x;
 3: lis_matrix_create(LIS_COMM_WORLD,&A); 
 4: lis_vector_create(LIS_COMM_WORLD,&b); 
 5: lis_vector_create(LIS_COMM_WORLD,&x); 
 6: lis_matrix_set_type(A,LIS_MATRIX_CSR); 
 7: lis_input(A,b,x,"matvec.mtx"); 
\end{verbatim}
\end{itemsquarebox}
\begin{itemsquarebox}[l]{Fortran (for serial, multithreaded and multiprocessing environments)}
\small
\begin{verbatim}
 1: LIS_MATRIX    A
 2: LIS_VECTOR    b,x
 3: call lis_matrix_create(LIS_COMM_WORLD,A,ierr) 
 4: call lis_vector_create(LIS_COMM_WORLD,b,ierr) 
 5: call lis_vector_create(LIS_COMM_WORLD,x,ierr) 
 6: call lis_matrix_set_type(A,LIS_MATRIX_CSR,ierr) 
 7: call lis_input(A,b,x,'matvec.mtx',ierr) 
\end{verbatim}
\end{itemsquarebox}
\\ \\
The content of the destination file {\tt matvec.mtx} is:
{\small
\begin{verbatim}
%%MatrixMarket matrix coordinate real general
4 4 10 1 0
1 2  1.0e+00
1 1  2.0e+00
2 3  1.0e+00
2 1  1.0e+00
2 2  2.0e+00
3 4  1.0e+00
3 2  1.0e+00
3 3  2.0e+00
4 4  2.0e+00
4 3  1.0e+00
1  0.0e+00
2  1.0e+00
3  2.0e+00
4  3.0e+00
\end{verbatim}
}

\noindent
{\bf Reading from External Files}

To input the matrix data for $A$ from an external file,
the following functions are used:
\begin{itemize}
\item \verb|C       LIS_INT lis_input_matrix(LIS_MATRIX A, char *filename)|
\item \verb|Fortran subroutine lis_input(LIS_MATRIX A,|\\
      \verb|         character filename, LIS_INTEGER ierr)|
\end{itemize}
{\tt filename} must be replaced with the file path.
The following file formats are supported:

\begin{itemize}
\item The Matrix Market format
\item The Harwell-Boeing format
\end{itemize}

To read the data for matrix $A$ and vectors $b$ and $x$ from
external files, the following functions are used:
\begin{itemize}
\item \verb|C       LIS_INT lis_input(LIS_MATRIX A, LIS_VECTOR b, LIS_VECTOR x, char *filename)|
\item \verb|Fortran subroutine lis_input(LIS_MATRIX A, LIS_VECTOR b, LIS_VECTOR x,|\\
      \verb|         character filename, LIS_INTEGER ierr)|
\end{itemize}
{\tt filename} must be replaced with the file path.
The following file formats are supported:

\begin{itemize}
\item The Extended Matrix Market format (extended to allow vector data)
\item The Harwell-Boeing format
\end{itemize}

\subsection{Solving Linear Equations}\label{subsec:solve}
A program to solve the linear equation $Ax = b$ with a specified 
solver is as follows: 
\begin{itemsquarebox}[l]{C (for serial, multithreaded and multiprocessing environments)}
\small
\begin{verbatim}
 1: LIS_MATRIX A; 
 2: LIS_VECTOR b,x; 
 3: LIS_SOLVER solver; 
 4:    
 5: /* Create matrix and vector */ 
 6:    
 7: lis_solver_create(&solver); 
 8: lis_solver_set_option("-i bicg -p none",solver); 
 9: lis_solver_set_option("-tol 1.0e-12",solver); 
10: lis_solve(A,b,x,solver); 
\end{verbatim}
\end{itemsquarebox}
\begin{itemsquarebox}[l]{Fortran (for serial, multithreaded and multiprocessing environments)}
\small
\begin{verbatim}
 1: LIS_MATRIX A 
 2: LIS_VECTOR b,x 
 3: LIS_SOLVER solver 
 4:    
 5: /* Create matrix and vector */ 
 6:    
 7: call lis_solver_create(solver,ierr) 
 8: call lis_solver_set_option('-i bicg -p none',solver,ierr) 
 9: call lis_solver_set_option('-tol 1.0e-12',solver,ierr) 
10: call lis_solve(A,b,x,solver,ierr) 
\end{verbatim}
\end{itemsquarebox}
\\ \\
\noindent
{\bf Creating Solvers}

To create a solver, the following functions are used:
\begin{itemize}
\item \verb|C       LIS_INT lis_solver_create(LIS_SOLVER *solver)|
\item \verb|Fortran subroutine lis_solver_create(LIS_SOLVER solver, LIS_INTEGER ierr) |
\end{itemize}
 \ \\ \\
\noindent
{\bf Specifying Options}

To specify options, 
the following functions are used:
\begin{itemize}
\item \verb|C       LIS_INT lis_solver_set_option(char *text, LIS_SOLVER solver)|
\item \verb|Fortran subroutine lis_solver_set_option(character text, LIS_SOLVER solver,|\\
      \verb|         LIS_INTEGER ierr)|
\end{itemize}
or
\begin{itemize}
\item \verb|C       LIS_INT lis_solver_set_optionC(LIS_SOLVER solver)|
\item \verb|Fortran subroutine lis_solver_set_optionC(LIS_SOLVER solver, LIS_INTEGER ierr)|
\end{itemize}
\verb|lis_solver_set_optionC| is a function that sets the options specified 
on the command line, and passes them to \verb|solver| when the program is run. 

The table below shows the available command line options, 
where \verb=-i {cg|1}= means \verb=-i cg= or \verb=-i 1= and \verb=-maxiter [1000]= indicates that \verb=-maxiter= defaults to $1,000$.
\\
\\
\begin{minipage}[t]{\textwidth}
\begin{center}
{\bf Options for Linear Solvers} (Default: \verb=-i bicg=) \\
\begin{tabular}{l|lll}\hline\hline
 Solver      & Option              &  Auxiliary Options  & \\ \hline
 CG          & \verb=-i {cg|1}=         &    \\ 
 BiCG        & \verb=-i {bicg|2}=       &    \\
 CGS         & \verb=-i {cgs|3}=        &    \\
 BiCGSTAB    & \verb=-i {bicgstab|4}=   &    \\
 BiCGSTAB(l) & \verb=-i {bicgstabl|5}=  & \verb=-ell [2]=      & The degree $l$ \\
 GPBiCG      & \verb=-i {gpbicg|6}=     &    \\
 TFQMR       & \verb=-i {tfqmr|7}=      &    \\
 Orthomin(m) & \verb=-i {orthomin|8}=   & \verb=-restart [40]= & The restart
 value $m$  \\
 GMRES(m)    & \verb=-i {gmres|9}=      & \verb=-restart [40]= & The restart value $m$  \\ 
 Jacobi      & \verb=-i {jacobi|10}=    &    \\
 Gauss-Seidel& \verb=-i {gs|11}=        &    \\
 SOR         & \verb=-i {sor|12}=       & \verb=-omega [1.9]=  & The relaxation coefficient $\omega$ ($0<\omega<2$) \\
 BiCGSafe    & \verb=-i {bicgsafe|13}=     &    \\
 CR          & \verb=-i {cr|14}=        &    \\ 
 BiCR        & \verb=-i {bicr|15}=      &    \\
 CRS         & \verb=-i {crs|16}=       &    \\
 BiCRSTAB    & \verb=-i {bicrstab|17}=  &    \\
 GPBiCR      & \verb=-i {gpbicr|18}=    &    \\
 BiCRSafe    & \verb=-i {bicrsafe|19}=  &    \\
 FGMRES(m)   & \verb=-i {fgmres|20}=    & \verb=-restart [40]= & The restart value $m$   \\ 
 IDR(s)      & \verb=-i {idrs|21}=      & \verb=-irestart [2]= & The restart
 value $s$  \\ 
 MINRES      & \verb=-i {minres|22}=    &    \\
\hline         
\end{tabular}
\end{center}
\end{minipage}
\\ \\
\begin{minipage}[t]{\textwidth}
\begin{center}
{\bf Options for Preconditioners} (Default: \verb=-p none=)\\
\begin{tabular}{l|lll}\hline\hline
Preconditioner   & Option           & Auxiliary Options \\ \hline
None     & \verb=-p {none|0}=    &   \\
Jacobi   & \verb=-p {jacobi|1}=  &     \\
ILU(k)   & \verb=-p {ilu|2}=     & \verb=-ilu_fill [0]=    & The fill level $k$ \\
SSOR     & \verb=-p {ssor|3}=    & \verb=-ssor_w [1.0]=    & The relaxation coefficient $\omega$ ($0<\omega<2$) \\
Hybrid   & \verb=-p {hybrid|4}=  & \verb=-hybrid_i [sor]=  & The linear solver\\
         &                       & \verb=-hybrid_maxiter [25]= & The
 maximum number of iterations \\
         &                       & \verb=-hybrid_tol [1.0e-3]= & The convergence criterion \\
         &                       & \verb=-hybrid_w [1.5]=      & The
 relaxation coefficient $\omega$ of the SOR \\
         &                       &                             & ($0<\omega<2$) \\
         &                       & \verb=-hybrid_ell [2]=      & The
 degree $l$ of the BiCGSTAB(l) \\
         &                       & \verb=-hybrid_restart [40]= & The
 restart values of the GMRES \\
         &                       &                             & and
 Orthomin \\
I+S      & \verb=-p {is|5}=      & \verb=-is_alpha [1.0]=  &  The
 parameter $\alpha$ of $I+\alpha S^{(m)}$ \\
         &                       & \verb=-is_m [3]=        & The
 parameter $m$ of $I+\alpha S^{(m)}$ \\
SAINV    & \verb=-p {sainv|6}=   & \verb=-sainv_drop [0.05]=    & The drop criterion\\
SA-AMG   & \verb=-p {saamg|7}=   & \verb=-saamg_unsym [false]=    &
 Select the unsymmetric version    \\
         &                       &                             & (The matrix structure must be symmetric)    \\
         &                       & \verb=-saamg_theta [0.05|0.12]= & The
 drop criterion $a^2_{ij}\le\theta^2|a_{ii}||a_{jj}|$ \\
         &                       &                             & (symmetric or unsymmetric) \\
Crout ILU& \verb=-p {iluc|8}=    & \verb=-iluc_drop [0.05]=    & The drop criterion    \\
         &                       & \verb=-iluc_rate [5.0]=     & The
 ratio of the maximum fill-in \\
ILUT     & \verb=-p {ilut|9}=    & \verb=-ilut_drop [0.05]=    & The drop criterion    \\
         &                       & \verb=-ilut_rate [5.0]=     & The
 ratio of the maximum fill-in \\
Additive Schwarz  & \verb=-adds true=   &  \verb=-adds_iter [1]= & The
 number of iterations   \\
\hline         
\end{tabular}
\end{center}
\end{minipage}
\\ \\
\begin{minipage}[t]{\textwidth}
\begin{center}
{\bf Other Options}\\
\begin{tabular}{l|ll}\hline\hline
Option &                          \\ \hline
\verb=-maxiter [1000]= & The maximum number of iterations         \\ 
\verb=-tol [1.0e-12]=  & The convergence criterion              \\
\verb=-print [0]=      & The display of the residual                 \\
                       & \verb=-print {none|0}     =  None \\
                       & \verb=-print {mem|1}      =  Save the residual history\\
                       & \verb=-print {out|2}      =  Display the residual history\\
                       & \verb=-print {all|3}      =  Save the residual history and display it on the screen\\
\verb=-scale [0]=      & The scaling \\
                       & (The result will overwrite the original matrix and vectors) \\
                       & \verb=-scale {none|0}     =  No scaling \\ 
                       & \verb=-scale {jacobi|1}   =  The Jacobi scaling $D^{-1}Ax=D^{-1}b$ \\
                       & \verb=                    =  ($D$ represents the diagonal of $A=(a_{ij})$)\\
                       & \verb=-scale {symm_diag|2}=  The diagonal scaling $D^{-1/2}AD^{-1/2}x=D^{-1/2}b$ \\
                       & \verb=                    =  ($D^{-1/2}$ represents the diagonal matrix with $1/\sqrt{a_{ii}}$ \\
                       & \verb=                    =   as the diagonal) \\ 
\verb=-initx_zeros [true]= & The behavior of the initial vector $x_{0}$  \\
                       & \verb=-initx_zeros {false|0}     =  Given values \\
                       & \verb=-initx_zeros {true|1}      =  All values are set to $0$ \\
\verb=-omp_num_threads [t]= & The number of threads        \\ 
                            & (\verb=t= represents the maximum number of
 threads) \\
\verb=-storage [0]=    & The matrix storage format \\
\verb=-storage_block [2]=& The block size of the BSR and BSC formats\\ 
\verb=-f [0]=          & The precision of the linear solver\\
                       & \verb=-f {double|0}       =  Double precision \\ 
                       & \verb=-f {quad|1}         =  Quadruple precision \\
\hline         
\end{tabular}
\end{center}
\end{minipage}
\\ \\ \\
\noindent
{\bf Solving Linear Equations}

To solve the linear equation $Ax = b$, the following functions are used:
\begin{itemize}
\item \verb|C       LIS_INT lis_solve(LIS_MATRIX A, LIS_VECTOR b, LIS_VECTOR x, LIS_SOLVER solver)|
\item \verb|Fortran subroutine lis_solve(LIS_MATRIX A, LIS_VECTOR b, LIS_VECTOR x,|\\
      \verb|         LIS_SOLVER solver, LIS_INTEGER ierr)|
\end{itemize}

\subsection{Solving Eigenvalue Problems}\label{subsec:solve}
A program to solve the eigenvalue problem $Ax = \lambda x$ with a specified 
solver is as follows: 
\begin{itemsquarebox}[l]{C (for serial, multithreaded and multiprocessing environments)}
\small
\begin{verbatim}
 1: LIS_MATRIX A; 
 2: LIS_VECTOR x; 
 3: LIS_REAL evalue; 
 4: LIS_ESOLVER esolver; 
 5: 
 6: /* Create matrix and vector */ 
 7: 
 8: lis_esolver_create(&esolver); 
 9: lis_esolver_set_option("-e ii -i bicg -p none",esolver); 
10: lis_esolver_set_option("-etol 1.0e-12 -tol 1.0e-12",esolver); 
11: lis_esolve(A,x,evalue,esolver); 
\end{verbatim}
\end{itemsquarebox}
\begin{itemsquarebox}[l]{Fortran (for serial, multithreaded and multiprocessing environments)}
\small
\begin{verbatim}
 1: LIS_MATRIX A 
 2: LIS_VECTOR x 
 3: LIS_REAL evalue
 4: LIS_ESOLVER esolver 
 5:
 6: /* Create matrix and vector */ 
 7:
 8: call lis_esolver_create(esolver,ierr) 
 9: call lis_esolver_set_option('-e ii -i bicg -p none',esolver,ierr) 
10: call lis_esolver_set_option('-etol 1.0e-12 -tol 1.0e-12',esolver,ierr) 
11: call lis_esolve(A,x,evalue,esolver,ierr) 
\end{verbatim}
\end{itemsquarebox}
\\ \\
\noindent
{\bf Creating Eigensolvers}

To create an eigensolver, the following functions are used:
\begin{itemize}
\item \verb|C       LIS_INT lis_esolver_create(LIS_ESOLVER *esolver)|
\item \verb|Fortran subroutine lis_esolver_create(LIS_ESOLVER esolver, LIS_INTEGER ierr) |
\end{itemize}
 \ \\ \\
\noindent
{\bf Specifying Options}

To specify options, 
the following functions are used:
\begin{itemize}
\item \verb|C       LIS_INT lis_esolver_set_option(char *text, LIS_ESOLVER esolver)|
\item \verb|Fortran subroutine lis_esolver_set_option(character text, LIS_ESOLVER esolver,|\\
      \verb|         LIS_INTEGER ierr)|
\end{itemize}
or
\begin{itemize}
\item \verb|C       LIS_INT lis_esolver_set_optionC(LIS_ESOLVER esolver)|
\item \verb|Fortran subroutine lis_esolver_set_optionC(LIS_ESOLVER esolver, LIS_INTEGER ierr)|
\end{itemize}
\verb|lis_esolver_set_optionC| is a function that sets the options specified 
in the command line, and passes them to \verb|esolver| when the program is run. 

The table below shows the available command line options, 
where \verb=-e {pi|1}= means \verb=-e pi= or \verb=-e 1= and \verb=-emaxiter [1000]= indicates 
that \verb=-emaxiter= defaults to $1,000$.
\\
\\
\begin{minipage}[t]{\textwidth}
\begin{center}
{\bf Options for Eigensolvers} (Default: \verb=-e pi=) \\
\begin{tabular}{l|lll}\hline\hline
 Eigensolver      & Option              &  Auxiliary Options  & \\ \hline
\hline
 Power                             & \verb=-e {pi|1}=        &    \\ 
 Inverse                           & \verb=-e {ii|2}=        & 
 \verb=-i [cg]= & The linear solver \\
 Approximate Inverse               & \verb=-e {aii|3}=       & 
 \verb=-i [cg]= & The linear solver \\
 Rayleigh Quotient                 & \verb=-e {rqi|4}=       &
 \verb=-i [cg]= & The linear solver \\
 Subspace                          & \verb=-e {si|5}=        &
 \verb=-ss [2]= & The size of the subspace \\
                                             &                         &
 \verb=-m [0]= & The mode number\\
 Lanczos                           & \verb=-e {li|6}=        &
 \verb=-ss [2]= & The size of the subspace \\
                                             &                         &
 \verb=-m [0]= & The mode number\\
 CG                          & \verb=-e {cg|7}=        &    \\
 CR                          & \verb=-e {cr|8}=        &    \\
\hline         
\end{tabular}
\end{center}
\end{minipage}
\\ \\
\begin{minipage}[t]{\textwidth}
\begin{center}
{\bf Options for Preconditioners} (Default: \verb=-p none=)\\
\begin{tabular}{l|lll}\hline\hline
Preconditioner   & Option           & Auxiliary Options \\ \hline
None     & \verb=-p {none|0}=    &   \\
Jacobi   & \verb=-p {jacobi|1}=  &     \\
ILU(k)   & \verb=-p {ilu|2}=     & \verb=-ilu_fill [0]=    & The fill level $k$ \\
SSOR     & \verb=-p {ssor|3}=    & \verb=-ssor_w [1.0]=    & The relaxation coefficient $\omega$ ($0<\omega<2$) \\
Hybrid   & \verb=-p {hybrid|4}=  & \verb=-hybrid_i [sor]=  & The linear solver\\
         &                       & \verb=-hybrid_maxiter [25]= & The
 maximum number of iterations \\
         &                       & \verb=-hybrid_tol [1.0e-3]= & The convergence criterion \\
         &                       & \verb=-hybrid_w [1.5]=      & The
 relaxation coefficient $\omega$ of the SOR \\
         &                       &                             & ($0<\omega<2$) \\
         &                       & \verb=-hybrid_ell [2]=      & The
 degree $l$ of the BiCGSTAB(l) \\
         &                       & \verb=-hybrid_restart [40]= & The
 restart values of the GMRES \\
         &                       &                             & and
 Orthomin \\
I+S      & \verb=-p {is|5}=      & \verb=-is_alpha [1.0]=  &  The
 parameter $\alpha$ of $I+\alpha S^{(m)}$ \\
         &                       & \verb=-is_m [3]=        & The
 parameter $m$ of $I+\alpha S^{(m)}$ \\
SAINV    & \verb=-p {sainv|6}=   & \verb=-sainv_drop [0.05]=    & The drop criterion\\
SA-AMG   & \verb=-p {saamg|7}=   & \verb=-saamg_unsym [false]=    &
 Select the unsymmetric version    \\
         &                       &                             & (The matrix structure must be symmetric)    \\
         &                       & \verb=-saamg_theta [0.05|0.12]= & The
 drop criterion $a^2_{ij}\le\theta^2|a_{ii}||a_{jj}|$ \\
         &                       &                             & (symmetric or unsymmetric) \\
Crout ILU& \verb=-p {iluc|8}=    & \verb=-iluc_drop [0.05]=    & The drop criterion    \\
         &                       & \verb=-iluc_rate [5.0]=     & The
 ratio of the maximum fill-in \\
ILUT     & \verb=-p {ilut|9}=    & \verb=-ilut_drop [0.05]=    & The drop criterion    \\
         &                       & \verb=-ilut_rate [5.0]=     & The
 ratio of the maximum fill-in \\
Additive Schwarz  & \verb=-adds true=   &  \verb=-adds_iter [1]= & The
 number of iterations   \\
\hline         
\end{tabular}
\end{center}
\end{minipage}
\\ \\
\begin{minipage}[t]{\textwidth}
\begin{center}
{\bf Other Options}\\
\begin{tabular}{l|ll}\hline\hline
Option &                          \\ \hline
\verb=-emaxiter [1000]= & The maximum number of iterations         \\ 
\verb=-etol [1.0e-12]=  & The convergence criterion              \\
\verb=-eprint [0]=      & The display of the residual                 \\
                       & \verb=-eprint {none|0}     =  None \\
                       & \verb=-eprint {mem|1}      =  Save the residual history\\
                       & \verb=-eprint {out|2}      =  Display the residual history\\
                       & \verb=-eprint {all|3}      =  Save the residual history and display it on the screen\\
\verb=-ie [ii]= & The inner eigensolver used in the Lanczos and Subspace\\
                       & \verb=-ie {pi|1}           =  The Power (the Subspace only)\\
                       & \verb=-ie {ii|2}           =  The Inverse \\
                       & \verb=-ie {aii|3}          =  The Approximate Inverse \\
                       & \verb=-ie {rqi|4}          =  The Rayleigh Quotient \\
\verb=-shift [0.0]= & The amount of the shift  \\
\verb=-initx_ones [true]= & The behavior of the initial vector $x_{0}$  \\
                       & \verb=-initx_ones {false|0}     =  Given values \\
                       & \verb=-initx_ones {true|1}      =  All values are set to $1$ \\
\verb=-omp_num_threads [t]= & The number of threads        \\ 
                            & (\verb=t= represents the maximum number of
 threads) \\
\verb=-estorage [0]=   & The matrix storage format \\
\verb=-estorage_block [2]=& The block size of the BSR and BSC formats\\ 
\verb=-ef [0]=         & The precision of the eigensolver\\
                       & \verb=-ef {double|0}       =  Double precision \\ 
                       & \verb=-ef {quad|1}         =  Quadruple precision \\
\hline         
\end{tabular}
\end{center}
\end{minipage}
\\ \\ \\
\noindent
{\bf Solving Eigenvalue Problems}

To solve the eigenvalue problem $Ax = \lambda x$, the following functions are used:
\begin{itemize}
\item \verb|C       LIS_INT lis_esolve(LIS_MATRIX A, LIS_VECTOR x,|\\ 
      \verb|         LIS_REAL evalue, LIS_ESOLVER esolver)|
\item \verb|Fortran subroutine lis_esolve(LIS_MATRIX A, LIS_VECTOR x,|\\
      \verb|         LIS_REAL evalue, LIS_ESOLVER esolver, LIS_INTEGER ierr)|
\end{itemize}

\subsection{Writing Programs}
\label{sec:testprog3}
The following are the programs for solving the linear equation $Ax = b$,
where matrix $A$ is a tridiagonal matrix 
\[
\left(
\begin{array}{ccccc}
2 & -1 &   &  &   \\
-1 & 2 & -1 &  &   \\
  & \ddots  & \ddots  & \ddots  &   \\
  &   & -1 & 2 & -1 \\
  &   &   & -1 & 2 \\
\end{array}
\right)
\]
of size $12$.
The the right-hand side vector $b$ is set such that the values of the elements of the solution $x$ are $1$. 
The program is located in the directory \verb|lis-($VERSION)/test|. 

\begin{itemsquarebox}[l]{Test program: test4.c}
{\small
\begin{verbatim}
 1: #include <stdio.h> 
 2: #include "lis.h" 
 3: main(LIS_INT argc, char *argv[]) 
 4: { 
 5:     LIS_INT i,n,gn,is,ie,iter; 
 6:     LIS_MATRIX A; 
 7:     LIS_VECTOR b,x,u; 
 8:     LIS_SOLVER solver; 
 9:     n = 12; 
10:     lis_initialize(&argc,&argv); 
11:     lis_matrix_create(LIS_COMM_WORLD,&A); 
12:     lis_matrix_set_size(A,0,n); 
13:     lis_matrix_get_size(A,&n,&gn) 
14:     lis_matrix_get_range(A,&is,&ie) 
15:     for(i=is;i<ie;i++) 
16:     { 
17:         if( i>0 ) lis_matrix_set_value(LIS_INS_VALUE,i,i-1,-1.0,A); 
18:         if( i<gn-1 ) lis_matrix_set_value(LIS_INS_VALUE,i,i+1,-1.0,A); 
19:         lis_matrix_set_value(LIS_INS_VALUE,i,i,2.0,A); 
20:     } 
21:     lis_matrix_set_type(A,LIS_MATRIX_CSR); 
22:     lis_matrix_assemble(A); 
23:  
24:     lis_vector_duplicate(A,&u); 
25:     lis_vector_duplicate(A,&b); 
26:     lis_vector_duplicate(A,&x); 
27:     lis_vector_set_all(1.0,u); 
28:     lis_matvec(A,u,b); 
29:  
30:     lis_solver_create(&solver); 
31:     lis_solver_set_optionC(solver); 
32:     lis_solve(A,b,x,solver); 
33:     lis_solver_get_iters(solver,&iter); 
34:     printf("iter = %d\n",iter); 
35:     lis_vector_print(x); 
36:     lis_matrix_destroy(A); 
37:     lis_vector_destroy(u); 
38:     lis_vector_destroy(b); 
39:     lis_vector_destroy(x); 
40:     lis_solver_destroy(solver); 
41:     lis_finalize(); 
42:     return 0; 
43: } 
}
\end{verbatim}
}
\end{itemsquarebox}
\begin{itemsquarebox}[l]{Test program: test4f.F}
{\small
\begin{verbatim}
 1:      implicit none
 2:      
 3:#include "lisf.h"
 4:
 5:      LIS_INTEGER      i,n,gn,is,ie,iter,ierr
 6:      LIS_MATRIX        A
 7:      LIS_VECTOR        b,x,u
 8:      LIS_SOLVER        solver
 9:      n  = 12
10:      call lis_initialize(ierr)
11:      call lis_matrix_create(LIS_COMM_WORLD,A,ierr)
12:      call lis_matrix_set_size(A,0,n,ierr)
13:      call lis_matrix_get_size(A,n,gn,ierr)
14:      call lis_matrix_get_range(A,is,ie,ierr)
15:      do i=is,ie-1
16:        if( i>1  ) call lis_matrix_set_value(LIS_INS_VALUE,i,i-1,-1.0d0,
17:     .                                        A,ierr)
18:        if( i<gn ) call lis_matrix_set_value(LIS_INS_VALUE,i,i+1,-1.0d0,
19:     .                                        A,ierr)
20:        call lis_matrix_set_value(LIS_INS_VALUE,i,i,2.0d0,A,ierr)
21:      enddo
22:      call lis_matrix_set_type(A,LIS_MATRIX_CSR,ierr)
23:      call lis_matrix_assemble(A,ierr)
24:
25:      call lis_vector_duplicate(A,u,ierr)
26:      call lis_vector_duplicate(A,b,ierr)
27:      call lis_vector_duplicate(A,x,ierr)
28:      call lis_vector_set_all(1.0d0,u,ierr)
29:      call lis_matvec(A,u,b,ierr)
30:
31:      call lis_solver_create(solver,ierr)
32:      call lis_solver_set_optionC(solver,ierr)
33:      call lis_solve(A,b,x,solver,ierr)
34:      call lis_solver_get_iters(solver,iter,ierr)
35:      write(*,*) 'iter = ',iter
36:      call lis_vector_print(x,ierr)
37:      call lis_matrix_destroy(A,ierr)
38:      call lis_vector_destroy(b,ierr)
39:      call lis_vector_destroy(x,ierr)
40:      call lis_vector_destroy(u,ierr)
41:      call lis_solver_destroy(solver,ierr)
42:      call lis_finalize(ierr)
43:
44:      stop
45:      end
\end{verbatim}
}
\end{itemsquarebox}

\subsection{Compiling and Linking}
Provided below is an example {\tt test4.c} located in
the directory \verb|lis-($VERSION)/test|, 
compiled on the SGI Altix 3700 using the Intel C/C++ Compiler (icc). 
Since the library includes some Fortran 90 codes 
when the SA-AMG preconditioner is selected, 
a Fortran 90 compiler must be used for the linking. 
The preprocessor macro \verb|USE_MPI| must be defined for the
multiprocessing environment. 
The preprocessor macros \verb|_LONGLONG| for C and \verb|LONGLONG| 
for Fortran must be defined when using the 64bit integer.

\begin{itemsquarebox}[l]{For the serial environment}
\small
{\bf Compiling}\\
\verb+      > icc -c -I($INSTALLDIR)/include test4.c+\\
{\bf Linking}\\
\verb+      > icc -o test4 test4.o -llis+\\
{\bf Linking (with SA-AMG)}\\
\verb+      > ifort -nofor_main -o test4 test4.o -llis+
\end{itemsquarebox}
\begin{itemsquarebox}[l]{For multithreaded environment}
\small
{\bf Compiling}\\
\verb+      > icc -c -openmp -I($INSTALLDIR)/include test4.c+\\
{\bf Linking}\\
\verb+      > icc -openmp -o test4 test4.o -llis+\\
{\bf Linking (with SA-AMG)}\\
\verb+      > ifort -nofor_main -openmp -o test4 test4.o -llis+
\end{itemsquarebox}
\begin{itemsquarebox}[l]{For multiprocessing environment}
\small
{\bf Compiling}\\
\verb+      > icc -c -DUSE_MPI -I($INSTALLDIR)/include test4.c+\\
{\bf Linking}\\
\verb+      > icc -o test4 test4.o -llis -lmpi+\\
{\bf Linking (with SA-AMG)}\\
\verb+      > ifort -nofor_main -o test4 test4.o -llis -lmpi+
\end{itemsquarebox}
\begin{itemsquarebox}[l]{For multithreaded and multiprocessing environments}
\small
{\bf Compiling}\\
\verb+      > icc -c -openmp -DUSE_MPI -I($INSTALLDIR)/include test4.c+\\
{\bf Linking}\\
\verb+      > icc -openmp -o test4 test4.o -llis -lmpi+\\
{\bf Linking (with SA-AMG)}\\
\verb+      > ifort -nofor_main -openmp -o test4 test4.o -llis -lmpi+
\end{itemsquarebox}

Provided below is an example \verb|test4f.F| located 
in the directory \verb|lis-($VERSION)/test|, compiled on the SGI Altix 3700 
using the Intel Fortran Compiler (ifort). 
Since compiler directives are used in the program, 
compiler option \verb|-fpp| is specified to use the preprocessor. 

\begin{itemsquarebox}[l]{For serial environment}
\small
{\bf Compiling}\\
\verb+      > ifort -c -fpp -I($INSTALLDIR)/include test4f.F+\\
{\bf Linking}\\
\verb+      > ifort -o test4f test4f.o -llis+\\
\end{itemsquarebox}
\begin{itemsquarebox}[l]{For multithreaded environment}
\small
{\bf Compiling}\\
\verb+      > ifort -c -fpp -openmp -I($INSTALLDIR)/include test4f.F+\\
{\bf Linking}\\
\verb+      > ifort -openmp -o test4f test4f.o -llis+\\
\end{itemsquarebox}
\begin{itemsquarebox}[l]{For multiprocessing environment}
\small
{\bf Compiling}\\
\verb+      > ifort -c -fpp -DUSE_MPI -I($INSTALLDIR)/include test4f.F+\\
{\bf Linking}\\
\verb+      > ifort -o test4f test4f.o -llis -lmpi+\\
\end{itemsquarebox}
\begin{itemsquarebox}[l]{For multithreaded and multiprocessing environments}
\small
{\bf Compiling}\\
\verb+      > ifort -c -fpp -openmp -DUSE_MPI -I($INSTALLDIR)/include test4f.F+\\
{\bf Linking}\\
\verb+      > ifort -openmp -o test4f test4f.o -llis -lmpi+\\
\end{itemsquarebox}

\subsection{Running}
The test programs \verb|test4| and \verb|test4f| 
in the directory \verb|lis-($VERSION)/test| are run as follows: \\\\
{\bf For serial environment}\\
\verb+      > ./test4 -i bicgstab+\\
{\bf For multithreaded environment}\\
\verb+      > env OMP_NUM_THREADS=2 ./test4 -i bicgstab +\\
{\bf For multiprocessing environment}\\
\verb+      > mpirun -np 2 ./test4 -i bicgstab +\\
{\bf For multithreaded and multiprocessing environment}\\
\verb+      > mpirun -np 2 env OMP_NUM_THREADS=2 ./test4 -i bicgstab +\\\\
The following results will be returned:

\begin{verbatim}
precision : double
solver    : BiCGSTAB 4
precon    : none
storage   : CSR
lis_solve : normal end

iter = 6
    0 1.000000e+000
    1 1.000000e+000
    2 1.000000e+000
    3 1.000000e+000
    4 1.000000e+000
    5 1.000000e+000
    6 1.000000e+000
    7 1.000000e+000
    8 1.000000e+000
    9 1.000000e+000
   10 1.000000e+000
   11 1.000000e+000
\end{verbatim}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\section{Quadruple Precision Operations}
\label{sec:quadruple}
\indent
Double precision operations sometimes require a large number of iterations 
because of the rounding error. Besides long double precision operations,
Lis supports "double double" precision operations, 
or quadruple precision operations by combining 
two double precision floating point numbers\cite{dd,qd}.
To use the double double precision with the same interface 
as the double precision operations, 
both the matrix and vectors are assumed to be double precision. 
Lis also supports the performance acceleration of the double double precision
operations with the SIMD instructions, such as
Intel's Streaming SIMD Extensions (SSE)\cite{quadlis}. 

\subsection{Using Quadruple Precision Operations}
\label{sec:testprog5}
The test program \verb|test5.c| solves a linear equation $Ax =b$, where $A$ is a Toeplitz matrix
\[
\left(
\begin{array}{cccccc}
2 & 1 &   &  &  & \\
0 & 2 & 1 &  &  & \\
\gamma & 0& 2 & 1 &  & \\
 & \ddots & \ddots & \ddots & \ddots & \\
 &  &   \gamma &0 &       2   & 1 \\
 &  &  &   \gamma & 0& 2 \\
\end{array}
\right).
\]
The right-hand vector is set such that the values of the elements 
of the solution are $1$. The value $n$ is the size of matrix $A$.
\verb|test5| with option {\tt -f} is run: \\
\\ \\
\noindent
{\bf Double precision}\\
By entering
\verb+      > ./test5 200 2.0 -f double+\\
the following results will be returned:

\begin{verbatim}
n = 200, gamma = 2.000000
initial vector x = 0
precision : double
solver    : BiCG 2
precon    : none
storage   : CSR
lis_solve : LIS_MAXITER(code=4)

BiCG: number of iterations     = 1001 (double = 1001, quad = 0)
BiCG: elapsed time             = 2.044368e-02 sec.
BiCG:   preconditioner         = 4.768372e-06 sec. 
BiCG:     matrix creation      = 4.768372e-06 sec.
BiCG:   linear solver          = 2.043891e-02 sec.
BiCG: relative residual 2-norm = 8.917591e+01
\end{verbatim}

\noindent
{\bf Quadruple precision}\\
By entering
\verb+      > ./test5 200 2.0 -f quad+\\
the following results will be returned:

\begin{verbatim}
n = 200, gamma = 2.000000
initial vector x = 0
precision : quad
solver    : BiCG 2
precon    : none
storage   : CSR
lis_solve : normal end

BiCG: number of iterations     = 230 (double = 230, quad = 0)
BiCG: elapsed time             = 2.267408e-02 sec.
BiCG:   preconditioner         = 4.549026e-04 sec. 
BiCG:     matrix creation      = 5.006790e-06 sec.
BiCG:   linear solver          = 2.221918e-02 sec.
BiCG: relative residual 2-norm = 6.499145e-11
\end{verbatim}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\section{Matrix Storage Formats}
\label{sec:storages}
This section describes the matrix storage formats supported by the library. 
Assume that the matrix row (column) number begins with $0$ and 
that the number of nonzero elements of matrix $A$ of size $n \times
n$ is $nnz$.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Compressed Sparse Row (CSR)
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{Compressed Sparse Row (CSR)}
The CSR format uses three arrays {\ttfamily ptr}, {\ttfamily index} and {\ttfamily value} to store data.
\begin{itemize}
\item {\ttfamily value} is a double precision array of length 
      $nnz$, which stores the nonzero elements of matrix $A$ along the row.
\item {\ttfamily index} is an integer array of length $nnz$,
      which stores the column numbers of the nonzero elements stored in
      the  array {\ttfamily value}.
\item {\ttfamily ptr} is an integer array of length $n+1$, which
      stores the starting points of the rows of the arrays {\ttfamily value} and {\ttfamily index}. 
\end{itemize}

\subsubsection{Creating Matrices (for Serial and Multithreaded Environments)}
The diagram on the right in Figure \ref{fig:storage01} shows how matrix $A$ in Figure \ref{fig:storage01} is stored in the CSR format. A program to create the matrix in the CSR format is as follows:
\begin{figure}[h]
{\centering 
\begin{minipage}{0.3\textwidth}
\begin{flushright}
$ \label{eq:mata}
A = \left(
\begin{array}{cccc}
11 &    &    &    \\
21 & 22 &    &    \\
   & 32 & 33 &    \\
41 &    & 43 & 44 \\
\end{array}\right)
$
\end{flushright}
\end{minipage}
\begin{minipage}{0.6\textwidth}
\begin{flushleft}
\includegraphics{storage01.eps} 
\end{flushleft}
\end{minipage}
\caption{Data structure of CSR format (for serial and multithreaded environments).}\label{fig:storage01}}
\end{figure}
\begin{itemsquarebox}[l]{For serial and multithreaded environments}
\small
\begin{verbatim}
 1: LIS_INT       n,nnz;
 2: LIS_INT       *ptr,*index;
 3: LIS_SCALAR    *value;
 4: LIS_MATRIX    A;
 5: n = 4; nnz = 8;
 6: ptr   = (LIS_INT *)malloc( (n+1)*sizeof(LIS_INT) );
 7: index = (LIS_INT *)malloc( nnz*sizeof(LIS_INT) );
 8: value = (LIS_SCALAR *)malloc( nnz*sizeof(LIS_SCALAR) );
 9: lis_matrix_create(0,&A);
10: lis_matrix_set_size(A,0,n);
11:
12: ptr[0] = 0; ptr[1] = 1; ptr[2] = 3; ptr[3] = 5; ptr[4] = 8;
13: index[0] =  0; index[1] =  0; index[2] =  1; index[3] =  1;
14: index[4] =  2; index[5] =  0; index[6] =  2; index[7] =  3;
15: value[0] = 11; value[1] = 21; value[2] = 22; value[3] = 32;
16: value[4] = 33; value[5] = 41; value[6] = 43; value[7] = 44;
17:
18:  lis_matrix_set_csr(nnz,ptr,index,value,A);
19:  lis_matrix_assemble(A);
\end{verbatim}
\end{itemsquarebox}

\newpage
\subsubsection{Creating Matrices (for Multiprocessing Environment)}
Figure \ref{fig:storage01_mpi} shows how matrix $A$ in 
Figure \ref{fig:storage01} is stored in the CSR format on two 
processing elements. A program to create the matrix in the CSR format 
on two processing elements is as follows:
\begin{figure}[h]
{\centering 
\includegraphics{storage01_mpi.eps} 
\caption{Data structure of CSR format (for multiprocessing environment).}\label{fig:storage01_mpi}}
\end{figure}
\begin{itemsquarebox}[l]{For multiprocessing environment}
\small
\begin{verbatim}
 1: LIS_INT       i,k,n,nnz,my_rank;
 2: LIS_INT       *ptr,*index;
 3: LIS_SCALAR    *value;
 4: LIS_MATRIX    A;
 5: MPI_Comm_rank(MPI_COMM_WORLD,&my_rank);
 6: if( my_rank==0 ) {n = 2; nnz = 3;}
 7: else             {n = 2; nnz = 5;}
 8: ptr   = (LIS_INT *)malloc( (n+1)*sizeof(LIS_INT) );
 9: index = (LIS_INT *)malloc( nnz*sizeof(LIS_INT) );
10: value = (LIS_SCALAR *)malloc( nnz*sizeof(LIS_SCALAR) );
11: lis_matrix_create(MPI_COMM_WORLD,&A);
12: lis_matrix_set_size(A,n,0);
13: if( my_rank==0 ) {
14:     ptr[0] = 0; ptr[1] = 1; ptr[2] = 3;
15:     index[0] =  0; index[1] =  0; index[2] =  1;
16:     value[0] = 11; value[1] = 21; value[2] = 22;}
17: else {
18:     ptr[0] = 0; ptr[1] = 2; ptr[2] = 5;
19:     index[0] =  1; index[1] =  2; index[2] =  0; index[3] =  2; index[4] =  3;
20:     value[0] = 32; value[1] = 33; value[2] = 41; value[3] = 43; value[4] = 44;}
21:  lis_matrix_set_csr(nnz,ptr,index,value,A);
22:  lis_matrix_assemble(A);
\end{verbatim}
\end{itemsquarebox}

\subsubsection{Associating Arrays}
To associate the arrays in the CSR format with matrix $A$, the following functions are used:
\begin{itemize}
\item \verb|C       LIS_INT lis_matrix_set_csr(LIS_INT nnz, LIS_INT row[], LIS_INT index[],|\\
      \verb| LIS_SCALAR value[], LIS_MATRIX A)|
\item \verb|Fortran subroutine lis_matrix_set_csr(LIS_INTEGER nnz, LIS_INTEGER row(),|\\
      \verb| LIS_INTEGER index(), LIS_SCALAR value(), LIS_MATRIX A, LIS_INTEGER ierr)|
\end{itemize}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Compressed Sparse Column (CSC)
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\subsection{Compressed Sparse Column (CSC)}
The CSS format uses three arrays {\ttfamily ptr}, {\ttfamily index} and
{\ttfamily value} to store data.
\begin{itemize}
\item {\ttfamily value} is a double precision array of length
      $nnz$, which stores the nonzero elements of matrix $A$ along the column.
\item {\ttfamily index} is an integer array of length $nnz$, which
      stores the row numbers of the nonzero elements stored in the
      array {\ttfamily value}.
\item {\ttfamily ptr} is an integer array of length $n+1$, which
      stores the starting points of the rows of the arrays {\ttfamily value} and {\ttfamily index}. 
\end{itemize}

\subsubsection{Creating Matrices (for Serial and Multithreaded Environments)}
The diagram on the right in Figure \ref{fig:storage02} shows how matrix $A$ in Figure \ref{fig:storage02} is stored in the CSC format. A program to create the matrix in the CSC format is as follows:

\begin{figure}[h]
{\centering 
\begin{minipage}{0.3\textwidth}
\begin{flushright}
$ 
A = \left(
\begin{array}{cccc}
11 &    &    &    \\
21 & 22 &    &    \\
   & 32 & 33 &    \\
41 &    & 43 & 44 \\
\end{array}\right)
$
\end{flushright}
\end{minipage}
\begin{minipage}{0.6\textwidth}
\begin{flushleft}
\includegraphics{storage02.eps} 
\end{flushleft}
\end{minipage}
\caption{Data structure of CSC format (for serial and multithreaded environments).}\label{fig:storage02}}
\end{figure}
\begin{itemsquarebox}[l]{For serial and multithreaded environments}
\small
\begin{verbatim}
 1: LIS_INT       n,nnz;
 2: LIS_INT       *ptr,*index;
 3: LIS_SCALAR    *value;
 4: LIS_MATRIX    A;
 5: n = 4; nnz = 8;
 6: ptr   = (LIS_INT *)malloc( (n+1)*sizeof(LIS_INT) );
 7: index = (LIS_INT *)malloc( nnz*sizeof(LIS_INT) );
 8: value = (LIS_SCALAR *)malloc( nnz*sizeof(LIS_SCALAR) );
 9: lis_matrix_create(0,&A);
10: lis_matrix_set_size(A,0,n);
11:
12: ptr[0] = 0; ptr[1] = 3; ptr[2] = 5; ptr[3] = 7; ptr[4] = 8;
13: index[0] =  0; index[1] =  1; index[2] =  3; index[3] =  1;
14: index[4] =  2; index[5] =  2; index[6] =  3; index[7] =  3;
15: value[0] = 11; value[1] = 21; value[2] = 41; value[3] = 22;
16: value[4] = 32; value[5] = 33; value[6] = 43; value[7] = 44;
17:
18:  lis_matrix_set_csc(nnz,ptr,index,value,A);
19:  lis_matrix_assemble(A);
\end{verbatim}
\end{itemsquarebox}

\newpage
\subsubsection{Creating Matrices (for Multiprocessing Environment)}
Figure \ref{fig:storage02_mpi} shows how matrix $A$ in Figure
\ref{fig:storage02} is stored on two processing elements. A program to create the
matrix in the CSC format on two processing elements is as follows:
\begin{figure}[h]
{\centering 
\includegraphics{storage02_mpi.eps} 
\caption{Data structure of CSC format (for multiprocessing environment).}\label{fig:storage02_mpi}}
\end{figure}
\begin{itemsquarebox}[l]{For multiprocessing environment}
\small
\begin{verbatim}
 1: LIS_INT       i,k,n,nnz,my_rank;
 2: LIS_INT       *ptr,*index;
 3: LIS_SCALAR    *value;
 4: LIS_MATRIX    A;
 5: MPI_Comm_rank(MPI_COMM_WORLD,&my_rank);
 6: if( my_rank==0 ) {n = 2; nnz = 3;}
 7: else             {n = 2; nnz = 5;}
 8: ptr   = (LIS_INT *)malloc( (n+1)*sizeof(LIS_INT) );
 9: index = (LIS_INT *)malloc( nnz*sizeof(LIS_INT) );
10: value = (LIS_SCALAR *)malloc( nnz*sizeof(LIS_SCALAR) );
11: lis_matrix_create(MPI_COMM_WORLD,&A);
12: lis_matrix_set_size(A,n,0);
13: if( my_rank==0 ) {
14:     ptr[0] = 0; ptr[1] = 3; ptr[2] = 5;
15:     index[0] =  0; index[1] =  1; index[2] =  3; index[3] =  1; index[4] =  2;
16:     value[0] = 11; value[1] = 21; value[2] = 41; value[3] = 22; value[4] = 32}
17: else {
18:     ptr[0] = 0; ptr[1] = 2; ptr[2] = 3;
19:     index[0] =  2; index[1] =  3; index[2] =  3;
20:     value[0] = 33; value[1] = 43; value[2] = 44;}
21:  lis_matrix_set_csc(nnz,ptr,index,value,A);
22:  lis_matrix_assemble(A);
\end{verbatim}
\end{itemsquarebox}

\subsubsection{Associating Arrays}
To associate the arrays in the CSC format with matrix $A$, the following functions are used:
\begin{itemize}
\item \verb|C       LIS_INT lis_matrix_set_csc(LIS_INT nnz, LIS_INT row[], LIS_INT index[],|\\
      \verb| LIS_SCALAR value[], LIS_MATRIX A)|
\item \verb|Fortran subroutine lis_matrix_set_csc(LIS_INTEGER nnz, LIS_INTEGER row(),|\\
      \verb| LIS_INTEGER index(), LIS_SCALAR value(), LIS_MATRIX A, LIS_INTEGER ierr)|
\end{itemize}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Modified Compressed Sparse Row (MSR)
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\subsection{Modified Compressed Sparse Row (MSR)}
The MSR format uses two arrays {\ttfamily index} and {\ttfamily value} to store data. 
Assume that $ndz$ represents the number of zero elements of the diagonal.
\begin{itemize}
\item {\ttfamily value} is a double precision array of length
      $nnz+ndz+1$, which stores the diagonal of matrix A down to the
      $n$-th element. The $n+1$-th element is not used. For the $n+2$-th
      and after, the values of the nonzero elements except the diagonal of matrix $A$ are stored along the row.
\item {\ttfamily index} is an integer array of length
      $nnz+ndz+1$, 
      which stores the starting points of the rows of the off-diagonal
      elements of matrix $A$ down to the $n+1$-th element. For the
      $n+2$-th and after, 
      it stores the row numbers of the off-diagonal elements of
      matrix $A$ stored in the array {\ttfamily value}.
\end{itemize}

\subsubsection{Creating Matrices (for Serial and Multithreaded Environments)}
The diagram on the right in Figure \ref{fig:storage03} shows how matrix A is stored in the MSR format. A program to create the matrix in the MSR format is as follows:
\begin{figure}[h]
{\centering 
\begin{minipage}{0.3\textwidth}
\begin{flushright}
$ 
A = \left(
\begin{array}{cccc}
11 &    &    &    \\
21 & 22 &    &    \\
   & 32 & 33 &    \\
41 &    & 43 & 44 \\
\end{array}\right)
$
\end{flushright}
\end{minipage}
\begin{minipage}{0.6\textwidth}
\begin{flushleft}
\includegraphics{storage03.eps} 
\end{flushleft}
\end{minipage}
\caption{Data structure of MSR format (for serial and multithreaded environments).}\label{fig:storage03}}
\end{figure}
\begin{itemsquarebox}[l]{For serial and multithreaded environments}
\small
\begin{verbatim}
 1: LIS_INT       n,nnz,ndz;
 2: LIS_INT       *index;
 3: LIS_SCALAR    *value;
 4: LIS_MATRIX    A;
 5: n = 4; nnz = 8; ndz = 0;
 6: index = (LIS_INT *)malloc( (nnz+ndz+1)*sizeof(LIS_INT) );
 7: value = (LIS_SCALAR *)malloc( (nnz+ndz+1)*sizeof(LIS_SCALAR) );
 8: lis_matrix_create(0,&A);
 9: lis_matrix_set_size(A,0,n);
10:
11: index[0] =  5; index[1] =  5; index[2] =  6; index[3] =  7;
12: index[4] =  9; index[5] =  0; index[6] =  1; index[7] =  0; index[8] =  2;
13: value[0] = 11; value[1] = 22; value[2] = 33; value[3] = 44;
14: value[4] =  0; value[5] = 21; value[6] = 32; value[7] = 41; value[8] = 43;
15:
16:  lis_matrix_set_msr(nnz,ndz,index,value,A);
17:  lis_matrix_assemble(A);
\end{verbatim}
\end{itemsquarebox}

\newpage
\subsubsection{Creating Matrices (for Multiprocessing Environment)}
Figure \ref{fig:storage03_mpi} shows how matrix $A$ in Figure
\ref{fig:storage03} is stored in the MSR format on two processing 
elements. A program to create the matrix in the MSR format on two processing element is as follows:
\begin{figure}[h]
{\centering 
\includegraphics{storage03_mpi.eps} 
\caption{Data structure of MSR format (for multiprocessing environment).}\label{fig:storage03_mpi}}
\end{figure}
\begin{itemsquarebox}[l]{For multiprocessing environment}
\small
\begin{verbatim}
 1: LIS_INT       i,k,n,nnz,ndz,my_rank;
 2: LIS_INT       *index;
 3: LIS_SCALAR    *value;
 4: LIS_MATRIX    A;
 5: MPI_Comm_rank(MPI_COMM_WORLD,&my_rank);
 6: if( my_rank==0 ) {n = 2; nnz = 3; ndz = 0;}
 7: else             {n = 2; nnz = 5; ndz = 0;}
 8: index = (LIS_INT *)malloc( (nnz+ndz+1)*sizeof(LIS_INT) );
 9: value = (LIS_SCALAR *)malloc( (nnz+ndz+1)*sizeof(LIS_SCALAR) );
10: lis_matrix_create(MPI_COMM_WORLD,&A);
11: lis_matrix_set_size(A,n,0);
12: if( my_rank==0 ) {
13:     index[0] =  3; index[1] =  3; index[2] =  4; index[3] =  0;
14:     value[0] = 11; value[1] = 22; value[2] =  0; value[3] = 21;}
15: else {
16:     index[0] =  3; index[1] =  4; index[2] =  6; index[3] =  1;
17:     index[4] =  0; index[5] =  2;
18:     value[0] = 33; value[1] = 44; value[2] =  0; value[3] = 32;
19:     value[4] = 41; value[5] = 43;}
20:  lis_matrix_set_msr(nnz,ndz,index,value,A);
21:  lis_matrix_assemble(A);
\end{verbatim}
\end{itemsquarebox}

\subsubsection{Associating Arrays}
To associate the arrays in the MSR format with matrix $A$, the following functions are used:
\begin{itemize}
\item \verb|C       LIS_INT lis_matrix_set_msr(LIS_INT nnz, LIS_INT ndz, LIS_INT index[],|\\
      \verb| LIS_SCALAR value[], LIS_MATRIX A)|
\item \verb|Fortran subroutine lis_matrix_set_msr(LIS_INTEGER nnz, LIS_INTEGER ndz,|\\
      \verb| LIS_INTEGER index(), LIS_SCALAR value(), LIS_MATRIX A, LIS_INTEGER ierr)|
\end{itemize}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Diagonal (DIA)
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\subsection{Diagonal (DIA)}
The DIA format uses two arrays {\ttfamily index} and {\ttfamily value} to store
data. Assume that $nnd$ represents the number of nonzero diagonal
elements of matrix $A$.
\begin{itemize}
\item {\ttfamily value} is a double precision array of length
      $nnd \times n$, which stores the values of the nonzero diagonal elements of matrix $A$.
\item {\ttfamily index} is an integer array of length $nnd$,
      which stores the offsets from the main diagonal.
\end{itemize}

For the multithreaded environment, the following modifications have been made:
the format uses two arrays {\ttfamily index} and {\ttfamily value} to store
data. Assume that $nprocs$ represents the number of threads.
$nnd_p$ is the number of nonzero diagonal elements of the partial matrix into which the row block of matrix $A$ is divided.
$maxnnd$ is the maximum value $nnd_p$.
\begin{itemize}
\item {\ttfamily value} is a double precision array of length 
      $maxnnd \times n$, which stores the values of the nonzero diagonal elements of matrix $A$.
\item {\ttfamily index} is an integer array of length $nprocs
      \times maxnnd$, which stores the offsets from the main diagonal.
\end{itemize}

\subsubsection{Creating Matrices (for Serial Environment)}
The diagram on the right in Figure \ref{fig:storage04} shows how matrix $A$
in Figure \ref{fig:storage04} is stored in the DIA format. A program to create the matrix in the DIA format is as follows:
\begin{figure}[h]
{\centering 
\begin{minipage}{0.3\textwidth}
\begin{flushright}
$ 
A = \left(
\begin{array}{cccc}
11 &    &    &    \\
21 & 22 &    &    \\
   & 32 & 33 &    \\
41 &    & 43 & 44 \\
\end{array}\right)
$
\end{flushright}
\end{minipage}
\begin{minipage}{0.6\textwidth}
\begin{flushleft}
\includegraphics{storage04.eps} 
\end{flushleft}
\end{minipage}
\caption{Data structure of DIA format (for serial environment).}\label{fig:storage04}}
\end{figure}
\begin{itemsquarebox}[l]{For serial environment}
\small
\begin{verbatim}
 1: LIS_INT       n,nnd;
 2: LIS_INT       *index;
 3: LIS_SCALAR    *value;
 4: LIS_MATRIX    A;
 5: n = 4; nnd = 3;
 6: index = (LIS_INT *)malloc( nnd*sizeof(LIS_INT) );
 7: value = (LIS_SCALAR *)malloc( n*nnd*sizeof(LIS_SCALAR) );
 8: lis_matrix_create(0,&A);
 9: lis_matrix_set_size(A,0,n);
10:
11: index[0] = -3; index[1] = -1; index[2] =  0;
12: value[0] =  0; value[1] =  0; value[2] =  0; value[3] = 41;
13: value[4] =  0; value[5] = 21; value[6] = 32; value[7] = 43;
14: value[8] = 11; value[9] = 22; value[10]= 33; value[11]= 44;
15:
16:  lis_matrix_set_dia(nnd,index,value,A);
17:  lis_matrix_assemble(A);
\end{verbatim}
\end{itemsquarebox}

\newpage
\subsubsection{Creating Matrices (for Multithreaded Environment)}
Figure \ref{fig:storage04_omp} shows how matrix $A$ in Figure \ref{fig:storage04} is stored in the DIA format on two threads. A program to create the matrix in the DIA format on two threads is as follows:
\begin{figure}[h]
{\centering 
\includegraphics{storage04_omp.eps} 
\caption{Data structure of DIA format (for multithreaded environment).}\label{fig:storage04_omp}}
\end{figure}
\begin{itemsquarebox}[l]{For multithreaded environment}
\small
\begin{verbatim}
 1: LIS_INT       n,maxnnd,nprocs;
 2: LIS_INT       *index;
 3: LIS_SCALAR    *value;
 4: LIS_MATRIX    A;
 5: n = 4; maxnnd = 3; nprocs = 2;
 6: index = (LIS_INT *)malloc( maxnnd*sizeof(LIS_INT) );
 7: value = (LIS_SCALAR *)malloc( n*maxnnd*sizeof(LIS_SCALAR) );
 8: lis_matrix_create(0,&A);
 9: lis_matrix_set_size(A,0,n);
10:
11: index[0] = -1; index[1] =  0; index[2] =  0; index[3] = -3; index[4] = -1; index[5] =  0;
12: value[0] =  0; value[1] = 21; value[2] = 11; value[3] = 22; value[4] =  0; value[5] =  0;
13: value[6] =  0; value[7] = 41; value[8] = 32; value[9] = 43; value[10]= 33; value[11]= 44;
14:
15:  lis_matrix_set_dia(maxnnd,index,value,A);
16:  lis_matrix_assemble(A);
\end{verbatim}
\end{itemsquarebox}

\newpage
\subsubsection{Creating Matrices (for Multiprocessing Environment)}
Figure \ref{fig:storage04_mpi} shows how matrix $A$ in Figure \ref{fig:storage04} is stored in the DIA format on two processing elements. A program to create the matrix in the DIA format on two processing elements is as follows:
\begin{figure}[h]
{\centering 
\includegraphics{storage04_mpi.eps} 
\caption{Data structure of DIA format (for multiprocessing environment).}\label{fig:storage04_mpi}}
\end{figure}
\begin{itemsquarebox}[l]{For multiprocessing environment}
\small
\begin{verbatim}
 1: LIS_INT       i,n,nnd,my_rank;
 2: LIS_INT       *index;
 3: LIS_SCALAR    *value;
 4: LIS_MATRIX    A;
 5: MPI_Comm_rank(MPI_COMM_WORLD,&my_rank);
 6: if( my_rank==0 ) {n = 2; nnd = 2;}
 7: else             {n = 2; nnd = 3;}
 8: index = (LIS_INT *)malloc( nnd*sizeof(LIS_INT) );
 9: value = (LIS_SCALAR *)malloc( n*nnd*sizeof(LIS_SCALAR) );
10: lis_matrix_create(MPI_COMM_WORLD,&A);
11: lis_matrix_set_size(A,n,0);
12: if( my_rank==0 ) {
13:     index[0] = -1; index[1] =  0;
14:     value[0] =  0; value[1] = 21; value[2] = 11; value[3] = 22;}
15: else {
16:     index[0] = -3; index[1] = -1; index[2] =  0;
17:     value[0] =  0; value[1] = 41; value[2] = 32; value[3] = 43; value[4] = 33;
18:     value[5] = 44;}
19:  lis_matrix_set_dia(nnd,index,value,A);
20:  lis_matrix_assemble(A);
\end{verbatim}
\end{itemsquarebox}

\subsubsection{Associating Arrays}
To associate the arrays in the DIA format with matrix $A$, the following functions are used:
\begin{itemize}
\item \verb|C       LIS_INT lis_matrix_set_dia(LIS_INT nnd, LIS_INT index[],|\\
      \verb| LIS_SCALAR value[], LIS_MATRIX A)|
\item \verb|Fortran subroutine lis_matrix_set_dia(LIS_INTEGER nnd, LIS_INTEGER index(),|\\
      \verb| LIS_SCALAR value(), LIS_MATRIX A, LIS_INTEGER ierr)|
\end{itemize}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Ellpack-Itpack Generalized Diagonal (ELL)
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\subsection{Ellpack-Itpack Generalized Diagonal (ELL)}
The ELL format uses two arrays {\ttfamily index} and {\ttfamily value} to store
data. Assume that $maxnzr$ is the maximum value of the number of nonzero elements in the rows of matrix $A$.
\begin{itemize}
\item {\ttfamily value} is a double precision array of length 
      $maxnzr \times n$, which stores the values of the nonzero elements of the rows
      of matrix $A$ along the column. The first column consists of
      the first nonzero elements of each row. If there is no nonzero elements to be stored, then $0$ is stored.
\item {\ttfamily index} is an integer array of length $maxnzr
      \times n$, which stores the column numbers of the nonzero
      elements stored in the array {\ttfamily value}. If the number of
      nonzero elements in the $i$-th row is $nnz$, then {\tt index[$nnz
      \times n + i$]} stores row number $i$.
\end{itemize}

\subsubsection{Creating Matrices (for Serial and Multithreaded Environments)}
The diagram on the right in Figure \ref{fig:storage05} shows how matrix $A$ in Figure \ref{fig:storage05} is stored in the ELL format. A program to create the matrix in the ELL format is as follows:
\begin{figure}[h]
{\centering 
\begin{minipage}{0.3\textwidth}
\begin{flushright}
$ 
A = \left(
\begin{array}{cccc}
11 &    &    &    \\
21 & 22 &    &    \\
   & 32 & 33 &    \\
41 &    & 43 & 44 \\
\end{array}\right)
$
\end{flushright}
\end{minipage}
\begin{minipage}{0.6\textwidth}
\begin{flushleft}
\includegraphics{storage05.eps} 
\end{flushleft}
\end{minipage}
\caption{Data structure of ELL format (for serial and multithreaded environments).}\label{fig:storage05}}
\end{figure}
\begin{itemsquarebox}[l]{For serial and multithreaded environments}
\small
\begin{verbatim}
 1: LIS_INT       n,maxnzr;
 2: LIS_INT       *index;
 3: LIS_SCALAR    *value;
 4: LIS_MATRIX    A;
 5: n = 4; maxnzr = 3;
 6: index = (LIS_INT *)malloc( n*maxnzr*sizeof(LIS_INT) );
 7: value = (LIS_SCALAR *)malloc( n*maxnzr*sizeof(LIS_SCALAR) );
 8: lis_matrix_create(0,&A);
 9: lis_matrix_set_size(A,0,n);
10:
11: index[0] =  0; index[1] =  0; index[2] =  1; index[3] =  0; index[4] =  0; index[5] =  1;
12: index[6] =  2; index[7] =  2; index[8] =  0; index[9] =  1; index[10]=  2; index[11]=  3;
13: value[0] = 11; value[1] = 21; value[2] = 32; value[3] = 41; value[4] =  0; value[5] = 22;
14: value[6] = 33; value[7] = 43; value[8] =  0; value[9] =  0; value[10]=  0; value[11]= 44;
15:
16:  lis_matrix_set_ell(maxnzr,index,value,A);
17:  lis_matrix_assemble(A);
\end{verbatim}
\end{itemsquarebox}

\newpage
\subsubsection{Creating Matrices (for Multiprocessing Environment)}
Figure \ref{fig:storage05_mpi} shows how matrix $A$ in Figure \ref{fig:storage05} is stored in the ELL format. A program to create the matrix in the ELL format on two processing elements is as follows:
\begin{figure}[h]
{\centering 
\includegraphics{storage05_mpi.eps} 
\caption{Data structure of ELL format (for multiprocessing environment).}\label{fig:storage05_mpi}}
\end{figure}
\begin{itemsquarebox}[l]{For multiprocessing environment}
\small
\begin{verbatim}
 1: LIS_INT       i,n,maxnzr,my_rank;
 2: LIS_INT       *index;
 3: LIS_SCALAR    *value;
 4: LIS_MATRIX    A;
 5: MPI_Comm_rank(MPI_COMM_WORLD,&my_rank);
 6: if( my_rank==0 ) {n = 2; maxnzr = 2;}
 7: else             {n = 2; maxnzr = 3;}
 8: index = (LIS_INT *)malloc( n*maxnzr*sizeof(LIS_INT) );
 9: value = (LIS_SCALAR *)malloc( n*maxnzr*sizeof(LIS_SCALAR) );
10: lis_matrix_create(MPI_COMM_WORLD,&A);
11: lis_matrix_set_size(A,n,0);
12: if( my_rank==0 ) {
13:     index[0] =  0; index[1] =  0; index[2] =  0; index[3] =  1;
14:     value[0] = 11; value[1] = 21; value[2] =  0; value[3] = 22;}
15: else {
16:     index[0] =  1; index[1] =  0; index[2] =  2; index[3] =  2; index[4] =  2;
17:     index[5] =  3;
18:     value[0] = 32; value[1] = 41; value[2] = 33; value[3] = 43; value[4] =  0;
19:     value[5] = 44;}
20:  lis_matrix_set_ell(maxnzr,index,value,A);
21:  lis_matrix_assemble(A);
\end{verbatim}
\end{itemsquarebox}

\subsubsection{Associating Arrays}
To associate an array required by the ELL format with matrix $A$, the following functions are used:
\begin{itemize}
\item \verb|C       LIS_INT lis_matrix_set_ell(LIS_INT maxnzr, LIS_INT index[],|\\
      \verb| LIS_SCALAR value[], LIS_MATRIX A)|
\item \verb|Fortran subroutine lis_matrix_set_ell(LIS_INTEGER maxnzr, LIS_INTEGER index(),|\\
      \verb| LIS_SCALAR value(), LIS_MATRIX A, LIS_INTEGER ierr)|
\end{itemize}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Jagged Diagonal (JAD)
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\subsection{Jagged Diagonal (JAD)}
The JAD format first sorts the nonzero elements of the rows in decreasing order of
size, and then stores them along the column. The JAD format uses four
arrays, {\ttfamily perm}, {\ttfamily ptr}, {\ttfamily index}, and {\ttfamily
value}, to store data. Assume that $maxnzr$ represents the maximum value
of the number of nonzero elements of matrix $A$.
\begin{itemize}
\item {\ttfamily perm} is an integer array of length $n$, which stores the sorted row numbers.
\item {\ttfamily value} is a double precision array of length 
      $nnz$, which stores the values of the jagged diagonal elements of the sorted matrix $A$. The
      first jagged diagonal consists of the values of the first nonzero elements of
      each row. The next jagged diagonal consists of the values of the second nonzero 
      elements, and so on. 
\item {\ttfamily index} is an integer array of length $nnz$,
      which stores the row numbers of the nonzero elements stored in
      the array {\ttfamily value}.
\item {\ttfamily ptr} is an integer array of length $maxnzr+1$,
      which stores the starting points of the jagged diagonal elements.
\end{itemize}

For the multithreaded environment, the following modifications have been made:
the format uses four arrays, {\ttfamily perm}, {\ttfamily ptr}, {\ttfamily
index}, and {\ttfamily value}, to store data. Assume that $nprocs$ is the
number of threads.
$maxnzr_p$ is the number of nonzero diagonal elements of the partial matrix into which the row block of matrix $A$ is divided.
$maxmaxnzr$ is the maximum value of $maxnzr_p$.
\begin{itemize}
\item {\ttfamily perm} is an integer array of length $n$, which stores the sorted row numbers.
\item {\ttfamily value} is a double precision array of length 
      $nnz$, which stores the values of the jagged diagonal elements of the sorted matrix $A$. The
      first jagged diagonal consists of the values of the first nonzero elements of
      each row. The next jagged diagonal consist of the values of the second
      nonzero elements of each row, and so on. 
\item {\ttfamily index} is an integer array of length $nnz$,
      which stores the row numbers of the nonzero elements stored in
      the array {\ttfamily value}.
\item {\ttfamily ptr} is an integer array of length $nprocs
      \times (maxmaxnzr + 1)$, which stores the starting points of the
      jagged diagonal elements.
\end{itemize}

\newpage
\subsubsection{Creating Matrices (for Serial Environment)}
The diagram on the right in Figure \ref{fig:storage06} shows how matrix $A$ in Figure \ref{fig:storage06} is stored in the JAD format. A program to create the matrix in the JAD format is as follows: 
\begin{figure}[h]
{\centering 
\begin{minipage}{0.3\textwidth}
\begin{flushright}
$ 
A = \left(
\begin{array}{cccc}
11 &    &    &    \\
21 & 22 &    &    \\
   & 32 & 33 &    \\
41 &    & 43 & 44 \\
\end{array}\right)
$
\end{flushright}
\end{minipage}
\begin{minipage}{0.6\textwidth}
\begin{flushleft}
\includegraphics{storage06.eps} 
\end{flushleft}
\end{minipage}
\caption{Data structure of JAD format (for serial environment).}\label{fig:storage06}}
\end{figure}
\begin{itemsquarebox}[l]{For serial environment}
\small
\begin{verbatim}
 1: LIS_INT       n,nnz,maxnzr;
 2: LIS_INT       *perm,*ptr,*index;
 3: LIS_SCALAR    *value;
 4: LIS_MATRIX    A;
 5: n = 4; nnz = 8; maxnzr = 3;
 6: perm  = (LIS_INT *)malloc( n*sizeof(LIS_INT) );
 7: ptr   = (LIS_INT *)malloc( (maxnzr+1)*sizeof(LIS_INT) );
 8: index = (LIS_INT *)malloc( nnz*sizeof(LIS_INT) );
 9: value = (LIS_SCALAR *)malloc( nnz*sizeof(LIS_SCALAR) );
10: lis_matrix_create(0,&A);
11: lis_matrix_set_size(A,0,n);
12:
13: perm[0] = 3; perm[1] = 1; perm[2] = 2; perm[3] = 0;
14: ptr[0]  = 0; ptr[1]  = 4; ptr[2]  = 7; ptr[3]  = 8;
15: index[0] =  0; index[1] =  0; index[2] =  1; index[3] =  0;
16: index[4] =  2; index[5] =  1; index[6] =  2; index[7] =  3;
17: value[0] = 41; value[1] = 21; value[2] = 32; value[3] = 11;
18: value[4] = 43; value[5] = 22; value[6] = 33; value[7] = 44;
19:
20:  lis_matrix_set_jad(nnz,maxnzr,perm,ptr,index,value,A);
21:  lis_matrix_assemble(A);
\end{verbatim}
\end{itemsquarebox}

\newpage
\subsubsection{Creating Matrices (for Multithreaded Environment)}
Figure \ref{fig:storage06_omp} shows how matrix $A$ in Figure \ref{fig:storage06} is stored in the JAD format on two threads. A program to create the matrix in the JAD format on two threads is as follows:
\begin{figure}[h]
{\centering 
\includegraphics{storage06_omp.eps} 
\caption{Data structure of JAD format (for multithreaded environment).}\label{fig:storage06_omp}}
\end{figure}
\begin{itemsquarebox}[l]{For multithreaded environment}
\small
\begin{verbatim}
 1: LIS_INT       n,nnz,maxmaxnzr,nprocs;
 2: LIS_INT       *perm,*ptr,*index;
 3: LIS_SCALAR    *value;
 4: LIS_MATRIX    A;
 5: n = 4; nnz = 8; maxmaxnzr = 3; nprocs = 2;
 6: perm  = (LIS_INT *)malloc( n*sizeof(LIS_INT) );
 7: ptr   = (LIS_INT *)malloc( nprocs*(maxmaxnzr+1)*sizeof(LIS_INT) );
 8: index = (LIS_INT *)malloc( nnz*sizeof(LIS_INT) );
 9: value = (LIS_SCALAR *)malloc( nnz*sizeof(LIS_SCALAR) );
10: lis_matrix_create(0,&A);
11: lis_matrix_set_size(A,0,n);
12:
13: perm[0] = 1; perm[1] = 0; perm[2] = 3; perm[3] = 2;
14: ptr[0]  = 0; ptr[1]  = 2; ptr[2]  = 3; ptr[3]  = 0;
15: ptr[4]  = 3; ptr[5]  = 5; ptr[6]  = 7; ptr[7]  = 8;
16: index[0] =  0; index[1] =  0; index[2] =  1; index[3] =  0;
17: index[4] =  1; index[5] =  2; index[6] =  2; index[7] =  3;
18: value[0] = 21; value[1] = 11; value[2] = 22; value[3] = 41;
19: value[4] = 32; value[5] = 43; value[6] = 33; value[7] = 44;
20:
21:  lis_matrix_set_jad(nnz,maxmaxnzr,perm,ptr,index,value,A);
22:  lis_matrix_assemble(A);
\end{verbatim}
\end{itemsquarebox}

\newpage
\subsubsection{Creating Matrices (for Multiprocessing Environment)}
Figure \ref{fig:storage06_mpi} shows how matrix $A$ in Figure \ref{fig:storage06} is stored in the JAD format on two processing elements. A program to create the matrix in the JAD format on two processing elements is as follows:
\begin{figure}[h]
{\centering 
\includegraphics{storage06_mpi.eps} 
\caption{Data structure of JAD format (for multiprocessing environment).}\label{fig:storage06_mpi}}
\end{figure}
\begin{itemsquarebox}[l]{For multiprocessing environment}
\small
\begin{verbatim}
 1: LIS_INT       i,n,nnz,maxnzr,my_rank;
 2: LIS_INT       *perm,*ptr,*index;
 3: LIS_SCALAR    *value;
 4: LIS_MATRIX    A;
 5: MPI_Comm_rank(MPI_COMM_WORLD,&my_rank);
 6: if( my_rank==0 ) {n = 2; nnz = 3; maxnzr = 2;}
 7: else             {n = 2; nnz = 5; maxnzr = 3;}
 8: perm  = (LIS_INT *)malloc( n*sizeof(LIS_INT) );
 9: ptr   = (LIS_INT *)malloc( (maxnzr+1)*sizeof(LIS_INT) );
10: index = (LIS_INT *)malloc( nnz*sizeof(LIS_INT) );
11: value = (LIS_SCALAR *)malloc( nnz*sizeof(LIS_SCALAR) );
12: lis_matrix_create(MPI_COMM_WORLD,&A);
13: lis_matrix_set_size(A,n,0);
14: if( my_rank==0 ) {
15:     perm[0] = 1; perm[1] = 0;
16:     ptr[0]  = 0; ptr[1]  = 2; ptr[2]  = 3;
17:     index[0] =  0; index[1] =  0; index[2] =  1;
18:     value[0] = 21; value[1] = 11; value[2] = 22;}
19: else {
20:     perm[0] = 3; perm[1] = 2;
21:     ptr[0]  = 0; ptr[1]  = 2; ptr[2]  = 4; ptr[3]  = 5;
22:     index[0] =  0; index[1] =  1; index[2] =  2; index[3] =  2; index[4] =  3;
23:     value[0] = 41; value[1] = 32; value[2] = 43; value[3] = 33; value[4] = 44;}
24:  lis_matrix_set_jad(nnz,maxnzr,perm,ptr,index,value,A);
25:  lis_matrix_assemble(A);
\end{verbatim}
\end{itemsquarebox}

\subsubsection{Associating Arrays}
To associate an array required by the JAD format with matrix $A$, the following functions are used:
\begin{itemize}
\item \verb|C       LIS_INT lis_matrix_set_jad(LIS_INT nnz, LIS_INT maxnzr, LIS_INT perm[],|\\
      \verb| LIS_INT ptr[], LIS_INT index[], LIS_SCALAR value[], LIS_MATRIX A)|
\item \verb|Fortran subroutine lis_matrix_set_jad(LIS_INTEGER nnz, LIS_INTEGER maxnzr,|\\
      \verb| LIS_INTEGER ptr(),integer index(), LIS_SCALAR value(), LIS_MATRIX A,|\\
      \verb| LIS_INTEGER ierr)|
\end{itemize}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Block Sparse Row (BSR)
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\subsection{Block Sparse Row (BSR)}
The BSR format breaks down matrix $A$ into partial matrices called blocks of size $r \times c$. 
The BSR format stores the nonzero blocks, in which at least one nonzero
element exists, in a format similar to that of CSR. 
Assume that $nr=n/r$ and $nnzb$ are the numbers of nonzero blocks of $A$. 
The BSR format uses three arrays {\ttfamily bptr}, {\ttfamily bindex} and {\ttfamily value} to store data.
\begin{itemize}
\item {\ttfamily value} is a double precision array of length
      $nnzb \times r \times c$, which stores the values of the elements of the nonzero blocks.
\item {\ttfamily bindex} is an integer array of length $nnzb$,
      which stores the block column numbers of the nonzero blocks.
\item {\ttfamily bptr} is an integer array of length $nr+1$,
      which stores the starting points of the block rows in the array {\ttfamily bindex}.
\end{itemize}

\subsubsection{Creating Matrices (for Serial and Multithreaded Environments)}
The diagram on the right in Figure \ref{fig:storage07} shows how matrix $A$ in Figure \ref{fig:storage07} is stored in the BSR format. A program to create the matrix in the BSR format is as follows:
\begin{figure}[h]
{\centering 
\begin{minipage}{0.3\textwidth}
\begin{flushright}
$ 
A = \left(
\begin{array}{cc|cc}
11 &    &    &    \\
21 & 22 &    &    \\ \hline
   & 32 & 33 &    \\
41 &    & 43 & 44 \\
\end{array}\right)
$
\end{flushright}
\end{minipage}
\begin{minipage}{0.6\textwidth}
\begin{flushleft}
\includegraphics{storage07.eps} 
\end{flushleft}
\end{minipage}
\caption{Data structure of BSR format (for serial and multithreaded environments).}\label{fig:storage07}}
\end{figure}
\begin{itemsquarebox}[l]{For serial and multithreaded environments}
\small
\begin{verbatim}
 1: LIS_INT       n,bnr,bnc,nr,nc,bnnz;
 2: LIS_INT       *bptr,*bindex;
 3: LIS_SCALAR    *value;
 4: LIS_MATRIX    A;
 5: n = 4; bnr = 2; bnc = 2; bnnz = 3; nr = (n-1)/bnr+1; nc = (n-1)/bnc+1;
 6: bptr   = (LIS_INT *)malloc( (nr+1)*sizeof(LIS_INT) );
 7: bindex = (LIS_INT *)malloc( bnnz*sizeof(LIS_INT) );
 8: value  = (LIS_SCALAR *)malloc( bnr*bnc*bnnz*sizeof(LIS_SCALAR) );
 9: lis_matrix_create(0,&A);
10: lis_matrix_set_size(A,0,n);
11:
12: bptr[0] = 0; bptr[1] = 1; bptr[2] = 3;
13: bindex[0] =  0; bindex[1] =  0; bindex[2] =  1;
14: value[0]  = 11; value[1] = 21; value[2] =  0; value[3] = 22;
15: value[4]  =  0; value[5] = 41; value[6] = 32; value[7] =  0;
16: value[8]  = 33; value[9] = 43; value[10]=  0; value[11]= 44;
17:
18:  lis_matrix_set_bsr(bnr,bnc,bnnz,bptr,bindex,value,A);
19:  lis_matrix_assemble(A);
\end{verbatim}
\end{itemsquarebox}

\newpage
\subsubsection{Creating Matrices (for Multiprocessing Environment)}
Figure \ref{fig:storage07_mpi} shows how matrix $A$ in Figure \ref{fig:storage07} is stored in the BSR format on two processing elements. A program to create the matrix in the BSR format on two processing elements is as follows:
\begin{figure}[h]
{\centering 
\includegraphics{storage07_mpi.eps} 
\caption{Data structure of BSR format (for multiprocessing environment).}\label{fig:storage07_mpi}}
\end{figure}
\begin{itemsquarebox}[l]{For multiprocessing environment}
\small
\begin{verbatim}
 1: LIS_INT       n,bnr,bnc,nr,nc,bnnz,my_rank;
 2: LIS_INT       *bptr,*bindex;
 3: LIS_SCALAR    *value;
 4: LIS_MATRIX    A;
 5: MPI_Comm_rank(MPI_COMM_WORLD,&my_rank);
 6: if( my_rank==0 ) {n = 2; bnr = 2; bnc = 2; bnnz = 1; nr = (n-1)/bnr+1; nc = (n-1)/bnc+1;}
 7: else             {n = 2; bnr = 2; bnc = 2; bnnz = 2; nr = (n-1)/bnr+1; nc = (n-1)/bnc+1;}
 8: bptr   = (LIS_INT *)malloc( (nr+1)*sizeof(LIS_INT) );
 9: bindex = (LIS_INT *)malloc( bnnz*sizeof(LIS_INT) );
10: value  = (LIS_SCALAR *)malloc( bnr*bnc*bnnz*sizeof(LIS_SCALAR) );
11: lis_matrix_create(MPI_COMM_WORLD,&A);
12: lis_matrix_set_size(A,n,0);
13: if( my_rank==0 ) {
14:     bptr[0] = 0; bptr[1] = 1;
15:     bindex[0] =  0;
16:     value[0]  = 11; value[1] = 21; value[2] =  0; value[3] = 22;}
17: else {
18:     bptr[0] = 0; bptr[1] = 2;
19:     bindex[0] =  0; bindex[1] =  1;
20:     value[0]  =  0; value[1]  = 41; value[2] = 32; value[3] =  0;
21:     value[4]  = 33; value[5]  = 43; value[6] =  0; value[7] = 44;}
22:  lis_matrix_set_bsr(bnr,bnc,bnnz,bptr,bindex,value,A);
23:  lis_matrix_assemble(A);
\end{verbatim}
\end{itemsquarebox}

\subsubsection{Associating Arrays}
To associate the arrays in the BSR format with matrix $A$, the following functions are used:
\begin{itemize}
\item \verb|C       LIS_INT lis_matrix_set_bsr(LIS_INT bnr, LIS_INT bnc, LIS_INT bnnz,|\\
      \verb| LIS_INT bptr[], LIS_INT bindex[], LIS_SCALAR value[], LIS_MATRIX A)|
\item \verb|Fortran subroutine lis_matrix_set_bsr(LIS_INTEGER bnr, LIS_INTEGER bnc,|\\
      \verb| LIS_INTEGER bnnz, LIS_INTEGER bptr(), LIS_INTEGER bindex(), LIS_SCALAR value(),|\\
      \verb| LIS_MATRIX A, LIS_INTEGER ierr)|
\end{itemize}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Block Sparse Column (BSC)
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\subsection{Block Sparse Column (BSC)}
The BSC format breaks down matrix $A$ into partial matrices called
blocks of size $r \times c$. The BSC format stores the nonzero blocks, in which at
least one nonzero element exists, in a format similar to that of CSC. Assume
that $nc=n/c$ and $nnzb$ are the numbers of the nonzero
blocks of A. The BSC format uses three arrays {\ttfamily bptr}, {\ttfamily bindex}
and {\ttfamily value} to store data. 
\begin{itemize}
\item {\ttfamily value} is a double precision array of length
      $nnzb \times r \times c$, which stores the values of the elements of the nonzero blocks.
\item {\ttfamily bindex} is an integer array of length $nnzb$,
      which stores the block row numbers of the nonzero blocks.
\item {\ttfamily bptr} is an integer array of length $nc+1$,
      which stores the starting points of the block columns in the array
      {\ttfamily bindex}.
\end{itemize}

\subsubsection{Creating Matrices (for Serial and Multithreaded Environments)}
The diagram on the right in Figure \ref{fig:storage08} shows how matrix $A$ in Figure \ref{fig:storage08} is stored in the BSC format. A program to create the matrix in the BSC format is as follows:
\begin{figure}[h]
{\centering 
\begin{minipage}{0.3\textwidth}
\begin{flushright}
$ 
A = \left(
\begin{array}{cc|cc}
11 &    &    &    \\
21 & 22 &    &    \\ \hline
   & 32 & 33 &    \\
41 &    & 43 & 44 \\
\end{array}\right)
$
\end{flushright}
\end{minipage}
\begin{minipage}{0.6\textwidth}
\begin{flushleft}
\includegraphics{storage08.eps} 
\end{flushleft}
\end{minipage}
\caption{Data structure of BSC format (for serial and multithreaded environments).}\label{fig:storage08}}
\end{figure}
\begin{itemsquarebox}[l]{For serial and multithreaded environments}
\small
\begin{verbatim}
 1: LIS_INT       n,bnr,bnc,nr,nc,bnnz;
 2: LIS_INT       *bptr,*bindex;
 3: LIS_SCALAR    *value;
 4: LIS_MATRIX    A;
 5: n = 4; bnr = 2; bnc = 2; bnnz = 3; nr = (n-1)/bnr+1; nc = (n-1)/bnc+1;
 6: bptr   = (LIS_INT *)malloc( (nc+1)*sizeof(LIS_INT) );
 7: bindex = (LIS_INT *)malloc( bnnz*sizeof(LIS_INT) );
 8: value  = (LIS_SCALAR *)malloc( bnr*bnc*bnnz*sizeof(LIS_SCALAR) );
 9: lis_matrix_create(0,&A);
10: lis_matrix_set_size(A,0,n);
11:
12: bptr[0] = 0; bptr[1] = 1; bptr[2] = 3;
13: bindex[0] =  0; bindex[1] =  1; bindex[2] =  1;
14: value[0]  = 11; value[1] = 21; value[2] =  0; value[3] = 22;
15: value[4]  =  0; value[5] = 41; value[6] = 32; value[7] =  0;
16: value[8]  = 33; value[9] = 43; value[10]=  0; value[11]= 44;
17:
18:  lis_matrix_set_bsc(bnr,bnc,bnnz,bptr,bindex,value,A);
19:  lis_matrix_assemble(A);
\end{verbatim}
\end{itemsquarebox}

\newpage
\subsubsection{Creating Matrices (for Multiprocessing Environment)}
Figure \ref{fig:storage08_mpi} shows how matrix $A$ in Figure
\ref{fig:storage08} is stored in the BSC format on two processing
elements. A program to create the matrix in the BSC format on two processing elements is as follows:
\begin{figure}[h]
{\centering 
\includegraphics{storage08_mpi.eps} 
\caption{Data structure of BSC format (for multiprocessing environment).}\label{fig:storage08_mpi}}
\end{figure}
\begin{itemsquarebox}[l]{For multiprocessing environment}
\small
\begin{verbatim}
 1: LIS_INT       n,bnr,bnc,nr,nc,bnnz,my_rank;
 2: LIS_INT       *bptr,*bindex;
 3: LIS_SCALAR    *value;
 4: LIS_MATRIX    A;
 5: MPI_Comm_rank(MPI_COMM_WORLD,&my_rank);
 6: if( my_rank==0 ) {n = 2; bnr = 2; bnc = 2; bnnz = 2; nr = (n-1)/bnr+1; nc = (n-1)/bnc+1;}
 7: else             {n = 2; bnr = 2; bnc = 2; bnnz = 1; nr = (n-1)/bnr+1; nc = (n-1)/bnc+1;}
 8: bptr   = (LIS_INT *)malloc( (nr+1)*sizeof(LIS_INT) );
 9: bindex = (LIS_INT *)malloc( bnnz*sizeof(LIS_INT) );
10: value  = (LIS_SCALAR *)malloc( bnr*bnc*bnnz*sizeof(LIS_SCALAR) );
11: lis_matrix_create(MPI_COMM_WORLD,&A);
12: lis_matrix_set_size(A,n,0);
13: if( my_rank==0 ) {
14:     bptr[0] = 0; bptr[1] = 2;
15:     bindex[0] =  0; bindex[1] =  1;
16:     value[0]  = 11; value[1]  = 21; value[2] =  0; value[3] = 22;
17:     value[4]  =  0; value[5]  = 41; value[6] = 32; value[7] =  0;}
18: else {
19:     bptr[0] = 0; bptr[1] = 1;
20:     bindex[0] =  1;
21:     value[0]  = 33; value[1]  = 43; value[2] =  0; value[3] = 44;}
22:  lis_matrix_set_bsc(bnr,bnc,bnnz,bptr,bindex,value,A);
23:  lis_matrix_assemble(A);
\end{verbatim}
\end{itemsquarebox}

\subsubsection{Associating Arrays}
To associate the arrays in the BSC format with matrix $A$, the following functions are used:
\begin{itemize}
\item \verb|C       LIS_INT lis_matrix_set_bsc(LIS_INT bnr, LIS_INT bnc, LIS_INT bnnz,|\\
      \verb| LIS_INT bptr[], LIS_INT bindex[], LIS_SCALAR value[], LIS_MATRIX A)|
\item \verb|Fortran subroutine lis_matrix_set_bsc(LIS_INTEGER bnr, LIS_INTEGER bnc,|\\
      \verb| LIS_INTEGER bnnz, LIS_INTEGER bptr(), LIS_INTEGER bindex(), LIS_SCALAR value(),|\\
      \verb| LIS_MATRIX A, LIS_INTEGER ierr)|
\end{itemize}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Variable Block Row (VBR)
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\subsection{Variable Block Row (VBR)}
The VBR format is the generalized version of the BSR format. The division points of the rows and columns are given by the arrays {\ttfamily row} and {\ttfamily col}. 
The VBR format stores the nonzero blocks (the blocks in which at least
one nonzero element exists) in a format similar to that of CSR. 
Assume that $nr$ and $nc$ are the numbers of row and column divisions, respectively, 
and that $nnzb$ denotes the number of nonzero blocks of A, 
and $nnz$ denotes the total number of elements of the nonzero blocks. 
The VBR format uses six arrays, {\ttfamily bptr}, {\ttfamily bindex}, {\ttfamily
row}, {\ttfamily col}, {\ttfamily ptr}, and {\ttfamily value}, to store data.
\begin{itemize}
\item {\ttfamily row} is an integer array of length $nr+1$, which
      stores the starting row number of the block rows.
\item {\ttfamily col} is an integer array of length $nc+1$, which
      stores the starting column number of the block columns.
\item {\ttfamily bindex} is an integer array of length $nnzb$,
      which stores the block column numbers of the nonzero blocks.
\item {\ttfamily bptr} is an integer array of length $nr+1$,
      which stores the starting points of the block rows in the array {\ttfamily bindex}.
\item {\ttfamily value} is a double precision array of length
      $nnz$, which stores the values of the elements of the nonzero blocks.
\item {\ttfamily ptr} is an integer array of length $nnzb+1$,
      which stores the starting points of the nonzero blocks in the
      array {\ttfamily value}.
\end{itemize}

\newpage
\subsubsection{Creating Matrices (for Serial and Multithreaded Environments)}
The diagram on the right in Figure \ref{fig:storage09} shows how matrix $A$ in Figure \ref{fig:storage09} is stored in the VBR format. A program to create the matrix in the VBR format is as follows:
\begin{figure}[h]
{\centering 
\begin{minipage}{0.3\textwidth}
\begin{flushright}
$ 
A = \left(
\begin{array}{c|cc|c}
11 &    &    &    \\ \hline
21 & 22 &    &    \\
   & 32 & 33 &    \\ \hline
41 &    & 43 & 44 \\
\end{array}\right)
$
\end{flushright}
\end{minipage}
\begin{minipage}{0.6\textwidth}
\begin{flushleft}
\includegraphics{storage09.eps} 
\end{flushleft}
\end{minipage}
\caption{Data structure of VBR format (for serial and multithreaded environments).}\label{fig:storage09}}
\end{figure}
\begin{itemsquarebox}[l]{For serial and multithreaded environments}
\small
\begin{verbatim}
 1: LIS_INT       n,nnz,nr,nc,bnnz;
 2: LIS_INT       *row,*col,*ptr,*bptr,*bindex;
 3: LIS_SCALAR    *value;
 4: LIS_MATRIX    A;
 5: n = 4; nnz = 11; bnnz = 6; nr = 3; nc = 3;
 6: bptr   = (LIS_INT *)malloc( (nr+1)*sizeof(LIS_INT) );
 7: row    = (LIS_INT *)malloc( (nr+1)*sizeof(LIS_INT) );
 8: col    = (LIS_INT *)malloc( (nc+1)*sizeof(LIS_INT) );
 9: ptr    = (LIS_INT *)malloc( (bnnz+1)*sizeof(LIS_INT) );
10: bindex = (LIS_INT *)malloc( bnnz*sizeof(LIS_INT) );
11: value  = (LIS_SCALAR *)malloc( nnz*sizeof(LIS_SCALAR) );
12: lis_matrix_create(0,&A);
13: lis_matrix_set_size(A,0,n);
14:
15: bptr[0] = 0; bptr[1] = 1; bptr[2] = 3; bptr[3] = 6;
16: row[0]  = 0; row[1]  = 1; row[2]  = 3; row[3] = 4;
17: col[0]  = 0; col[1]  = 1; col[2]  = 3; col[3] = 4;
18: bindex[0] =  0; bindex[1] =  0; bindex[2] =  1; bindex[3] =  0;
19: bindex[4] =  1; bindex[5] =  2;
20: ptr[0]    =  0; ptr[1]    =  1; ptr[2]    =  3; ptr[3]    =  7;
21: ptr[4]    =  8; ptr[5]    = 10; ptr[6]    = 11;
22: value[0]  = 11; value[1]  = 21; value[2]  =  0; value[3]  = 22;
23: value[4]  = 32; value[5]  =  0; value[6]  = 33; value[7]  = 41;
24: value[8]  =  0; value[9]  = 43; value[10] = 44;
25:
26:  lis_matrix_set_vbr(nnz,nr,nc,bnnz,row,col,ptr,bptr,bindex,value,A);
27:  lis_matrix_assemble(A);
\end{verbatim}
\end{itemsquarebox}

\newpage
\subsubsection{Creating Matrices (for Multiprocessing Environment)}
Figure \ref{fig:storage09_mpi} shows how matrix $A$ in Figure \ref{fig:storage09} is stored in the VBR format on two processing elements. A program to create the matrix in the VBR format on two processing elements is as follows:
\begin{figure}[h]
{\centering 
\includegraphics{storage09_mpi.eps} 
\caption{Data structure of VBR format (for multiprocessing environment).}\label{fig:storage09_mpi}}
\end{figure}
\begin{itemsquarebox}[l]{For multiprocessing environment}
\small
\begin{verbatim}
 1: LIS_INT       n,nnz,nr,nc,bnnz,my_rank;
 2: LIS_INT       *row,*col,*ptr,*bptr,*bindex;
 3: LIS_SCALAR    *value;
 4: LIS_MATRIX    A;
 5: MPI_Comm_rank(MPI_COMM_WORLD,&my_rank);
 6: if( my_rank==0 ) {n = 2; nnz = 7; bnnz = 3; nr = 2; nc = 3;}
 7: else             {n = 2; nnz = 4; bnnz = 3; nr = 1; nc = 3;}
 8: bptr   = (LIS_INT *)malloc( (nr+1)*sizeof(LIS_INT) );
 9: row    = (LIS_INT *)malloc( (nr+1)*sizeof(LIS_INT) );
10: col    = (LIS_INT *)malloc( (nc+1)*sizeof(LIS_INT) );
11: ptr    = (LIS_INT *)malloc( (bnnz+1)*sizeof(LIS_INT) );
12: bindex = (LIS_INT *)malloc( bnnz*sizeof(LIS_INT) );
13: value  = (LIS_SCALAR *)malloc( nnz*sizeof(LIS_SCALAR) );
14: lis_matrix_create(MPI_COMM_WORLD,&A);
15: lis_matrix_set_size(A,n,0);
16: if( my_rank==0 ) {
17:     bptr[0] = 0; bptr[1] = 1; bptr[2] = 3;
18:     row[0]  = 0; row[1]  = 1; row[2]  = 3;
19:     col[0]  = 0; col[1]  = 1; col[2]  = 3; col[3] = 4;
20:     bindex[0] =  0; bindex[1] =  0; bindex[2] =  1;
21:     ptr[0]    =  0; ptr[1]    =  1; ptr[2]    =  3; ptr[3]    =  7;
22:     value[0]  = 11; value[1] = 21; value[2] =  0; value[3] = 22;
23:     value[4]  = 32; value[5] =  0; value[6] = 33;}
24: else {
25:     bptr[0] = 0; bptr[1] = 3;
26:     row[0]  = 3; row[1]  = 4;
27:     col[0]  = 0; col[1]  = 1; col[2]  = 3; col[3] = 4;
28:     bindex[0] =  0; bindex[1] =  1; bindex[2] =  2;
29:     ptr[0]    =  0; ptr[1]    =  1; ptr[2]    =  3; ptr[3]    =  4;
30:     value[0]  = 41; value[1]  =  0; value[2]  = 43; value[3]  = 44;}
31:  lis_matrix_set_vbr(nnz,nr,nc,bnnz,row,col,ptr,bptr,bindex,value,A);
32:  lis_matrix_assemble(A);
\end{verbatim}
\end{itemsquarebox}

\subsubsection{Associating Arrays}
To associate the arrays in the VBR format with matrix $A$, the following functions are used:
\begin{itemize}
\item \verb|C       LIS_INT lis_matrix_set_vbr(LIS_INT nnz, LIS_INT nr, LIS_INT nc,|\\
      \verb| LIS_INT bnnz, LIS_INT row[], LIS_INT col[], LIS_INT ptr[], LIS_INT bptr[],|\\
      \verb| LIS_INT bindex[], LIS_SCALAR value[], LIS_MATRIX A)|
\item \verb|Fortran subroutine lis_matrix_set_vbr(LIS_INTEGER nnz, LIS_INTEGER nr,|\\
      \verb| LIS_INTEGER nc, LIS_INTEGER bnnz, LIS_INTEGER row(), LIS_INTEGER col(),|\\
      \verb| LIS_INTEGER ptr(), LIS_INTEGER bptr(), LIS_INTEGER bindex(), |\\
      \verb| LIS_SCALAR value(), LIS_MATRIX A, LIS_INTEGER ierr)|
\end{itemize}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Coordinate (COO)
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\subsection{Coordinate (COO)}
The COO format uses three arrays {\ttfamily row}, {\ttfamily col} and {\ttfamily value} to store data.
\begin{itemize}
\item {\ttfamily value} is a double precision array of length
      $nnz$, which stores the values of the nonzero elements.
\item {\ttfamily row} is an integer array of length $nnz$, which stores the row numbers of the nonzero elements.
\item {\ttfamily col} is an integer array of length $nnz$, which stores the column numbers of the nonzero elements.
\end{itemize}

\subsubsection{Creating Matrices (for Serial and Multithreaded Environments)}
The diagram on the right in Figure \ref{fig:storage10} shows how matrix $A$ in Figure \ref{fig:storage10} is stored in the COO format. A program to create the matrix in the COO format is as follows:
\begin{figure}[h]
{\centering 
\begin{minipage}{0.3\textwidth}
\begin{flushright}
$ 
A = \left(
\begin{array}{cccc}
11 &    &    &    \\
21 & 22 &    &    \\
   & 32 & 33 &    \\
41 &    & 43 & 44 \\
\end{array}\right)
$
\end{flushright}
\end{minipage}
\begin{minipage}{0.6\textwidth}
\begin{flushleft}
\includegraphics{storage10.eps} 
\end{flushleft}
\end{minipage}
\caption{Data structure of COO format (for serial and multithreaded environments).}\label{fig:storage10}}
\end{figure}
\begin{itemsquarebox}[l]{For serial and multithreaded environments}
\small
\begin{verbatim}
 1: LIS_INT       n,nnz;
 2: LIS_INT       *row,*col;
 3: LIS_SCALAR    *value;
 4: LIS_MATRIX    A;
 5: n = 4; nnz = 8;
 6: row   = (LIS_INT *)malloc( nnz*sizeof(LIS_INT) );
 7: col   = (LIS_INT *)malloc( nnz*sizeof(LIS_INT) );
 8: value = (LIS_SCALAR *)malloc( nnz*sizeof(LIS_SCALAR) );
 9: lis_matrix_create(0,&A);
10: lis_matrix_set_size(A,0,n);
11:
12: row[0] = 0; row[1] = 1; row[2] = 3; row[3] = 1;
13: row[4] = 2; row[5] = 2; row[6] = 3; row[7] = 3;
14: col[0] = 0; col[1] = 0; col[2] = 0; col[3] = 1;
15: col[4] = 1; col[5] = 2; col[6] = 2; col[7] = 3;
16: value[0] = 11; value[1] = 21; value[2] = 41; value[3] = 22;
17: value[4] = 32; value[5] = 33; value[6] = 43; value[7] = 44;
18:
19:  lis_matrix_set_coo(nnz,row,col,value,A);
20:  lis_matrix_assemble(A);
\end{verbatim}
\end{itemsquarebox}

\newpage
\subsubsection{Creating Matrices (for Multiprocessing Environment)}
Figure \ref{fig:storage10_mpi} shows how matrix $A$ in Figure
\ref{fig:storage10} is stored in the COO format on two processing
elements. A program to create the matrix in the COO format on two processing elements is as follows:
\begin{figure}[h]
{\centering 
\includegraphics{storage10_mpi.eps} 
\caption{Data structure of COO format (for multiprocessing environment).}\label{fig:storage10_mpi}}
\end{figure}
\begin{itemsquarebox}[l]{For multiprocessing environment}
\small
\begin{verbatim}
 1: LIS_INT       n,nnz,my_rank;
 2: LIS_INT       *row,*col;
 3: LIS_SCALAR    *value;
 4: LIS_MATRIX    A;
 5: MPI_Comm_rank(MPI_COMM_WORLD,&my_rank);
 6: if( my_rank==0 ) {n = 2; nnz = 3;}
 7: else             {n = 2; nnz = 5;}
 8: row   = (LIS_INT *)malloc( nnz*sizeof(LIS_INT) );
 9: col   = (LIS_INT *)malloc( nnz*sizeof(LIS_INT) );
10: value = (LIS_SCALAR *)malloc( nnz*sizeof(LIS_SCALAR) );
11: lis_matrix_create(MPI_COMM_WORLD,&A);
12: lis_matrix_set_size(A,n,0);
13: if( my_rank==0 ) {
14:     row[0] = 0; row[1] = 1; row[2] = 1;
15:     col[0] = 0; col[1] = 0; col[2] = 1;
16:     value[0] = 11; value[1] = 21; value[2] = 22;}
17: else {
18:     row[0] = 3; row[1] = 2; row[2] = 2; row[3] = 3; row[4] = 3;
19:     col[0] = 0; col[1] = 1; col[2] = 2; col[3] = 2; col[4] = 3;
20:     value[0] = 41; value[1] = 32; value[2] = 33; value[3] = 43; value[4] = 44;}
21:  lis_matrix_set_coo(nnz,row,col,value,A);
22:  lis_matrix_assemble(A);
\end{verbatim}
\end{itemsquarebox}

\subsubsection{Associating Arrays}
To associate the arrays in the COO format with matrix $A$, the following functions are used:
\begin{itemize}
\item \verb|C       LIS_INT lis_matrix_set_coo(LIS_INT nnz, LIS_INT row[], LIS_INT col[],|\\
      \verb| LIS_SCALAR value[], LIS_MATRIX A)|
\item \verb|Fortran subroutine lis_matrix_set_coo(LIS_INTEGER nnz, LIS_INTEGER row(),|\\
      \verb| LIS_INTEGER col(), LIS_SCALAR value(), LIS_MATRIX A, LIS_INTEGER ierr)|
\end{itemize}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Dense (DNS)
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\subsection{Dense (DNS)}
The DNS format uses one array {\ttfamily value} to store data.
\begin{itemize}
\item {\ttfamily value} is a double precision array of length $n
      \times n$, which stores the values of the elements with priority given to the columns.
\end{itemize}

\subsubsection{Creating Matrices (for Serial and Multithreaded Environments)}
The right diagram in Figure \ref{fig:storage11} shows how matrix $A$ in Figure \ref{fig:storage11} is stored in the DNS format. A program to create the matrix in the DNS format is as follows:
\begin{figure}[h]
{\centering 
\begin{minipage}{0.3\textwidth}
\begin{flushright}
$ 
A = \left(
\begin{array}{cccc}
11 &    &    &    \\
21 & 22 &    &    \\
   & 32 & 33 &    \\
41 &    & 43 & 44 \\
\end{array}\right)
$
\end{flushright}
\end{minipage}
\begin{minipage}{0.6\textwidth}
\begin{flushleft}
\includegraphics{storage11.eps} 
\end{flushleft}
\end{minipage}
\caption{Data structure of DNS format (for serial and multithreaded environments).}\label{fig:storage11}}
\end{figure}
\begin{itemsquarebox}[l]{For serial and multithreaded environments}
\small
\begin{verbatim}
 1: LIS_INT       n;
 2: LIS_SCALAR    *value;
 3: LIS_MATRIX    A;
 4: n = 4;
 5: value = (LIS_SCALAR *)malloc( n*n*sizeof(LIS_SCALAR) );
 6: lis_matrix_create(0,&A);
 7: lis_matrix_set_size(A,0,n);
 8:
 9: value[0] = 11; value[1] = 21; value[2] =  0; value[3] = 41;
10: value[4] =  0; value[5] = 22; value[6] = 32; value[7] =  0;
11: value[8] =  0; value[9] =  0; value[10]= 33; value[11]= 43;
12: value[12]=  0; value[13]=  0; value[14]=  0; value[15]= 44;
13:
14:  lis_matrix_set_dns(value,A);
15:  lis_matrix_assemble(A);
\end{verbatim}
\end{itemsquarebox}

\newpage
\subsubsection{Creating Matrices (for Multiprocessing Environment)}
Figure \ref{fig:storage11_mpi} shows how matrix $A$ in Figure
\ref{fig:storage11} is stored in the DNS format on two
processing elements. A program to create the matrix in the DNS format on
two processing elements is as follows:
\begin{figure}[h]
{\centering 
\includegraphics{storage11_mpi.eps} 
\caption{Data structure of DNS format (for multiprocessing environment).}\label{fig:storage11_mpi}}
\end{figure}
\begin{itemsquarebox}[l]{For multiprocessing environment}
\small
\begin{verbatim}
 1: LIS_INT       n,my_rank;
 2: LIS_SCALAR    *value;
 3: LIS_MATRIX    A;
 4: MPI_Comm_rank(MPI_COMM_WORLD,&my_rank);
 5: if( my_rank==0 ) {n = 2;}
 6: else             {n = 2;}
 7: value = (LIS_SCALAR *)malloc( n*n*sizeof(LIS_SCALAR) );
 8: lis_matrix_create(MPI_COMM_WORLD,&A);
 9: lis_matrix_set_size(A,n,0);
10: if( my_rank==0 ) {
11:     value[0] = 11; value[1] = 21; value[2] =  0; value[3] = 22;
12:     value[4] =  0; value[5] =  0; value[6] =  0; value[7] =  0;}
13: else {
14:     value[0] =  0; value[1] = 41; value[2] = 32; value[3] =  0;
15:     value[4] = 33; value[5] = 43; value[6] =  0; value[7] = 44;}
16:  lis_matrix_set_dns(value,A);
17:  lis_matrix_assemble(A);
\end{verbatim}
\end{itemsquarebox}

\subsubsection{Associating Arrays}
To associate the arrays in the DNS format with matrix $A$, the following functions are used:
\begin{itemize}
\item \verb|C       LIS_INT lis_matrix_set_dns(LIS_SCALAR value[], LIS_MATRIX A)|
\item \verb|Fortran subroutine lis_matrix_set_dns(LIS_SCALAR value(), LIS_MATRIX A,|\\
      \verb| LIS_INTEGER ierr)|
\end{itemize}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\section{Functions}\label{sec:func}
This section describes the functions which can be employed by the user.
The return codes of the functions in C and the values of {\tt ierr} in Fortran are: 
\\ \\ 
\noindent
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt LIS\_SUCCESS(0)] ~~~~~Normal termination
\item[\tt LIS\_ILL\_OPTION(1)] ~~~~~Illegal option
\item[\tt LIS\_BREAKDOWN(2)] ~~~~~Breakdown
\item[\tt LIS\_OUT\_OF\_MEMORY(3)] ~~~~~Out of working memory
\item[\tt LIS\_MAXITER(4)] ~~~~~Maximum number of iterations
\item[\tt LIS\_NOT\_IMPLEMENTED(5)] ~~~~~Not implemented
\item[\tt LIS\_ERR\_FILE\_IO(6)] ~~~~~File I/O error
\end{namelist}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\subsection{Operating Vector Elements}
Assume that the size of vector $v$ is $global\_n$ and that the size
of the partial vectors stored on $nprocs$ processing elements 
is $local\_n$. $global\_n$ and $local\_n$ 
are called the global size and the local size, respectively.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{lis\_vector\_create}
\begin{screen}
\verb|C       LIS_INT lis_vector_create(LIS_Comm comm, LIS_VECTOR *v)|\\
\verb|Fortran subroutine lis_vector_create(LIS_Comm comm, LIS_VECTOR v, LIS_INTEGER ierr)| 
\end{screen}
\\ \\
\noindent
{\bf Description}\\
\indent
Create vector $v$.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt LIS\_Comm] The MPI communicator
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt v] The vector
\item[\tt ierr] The return code
\end{namelist}
{\bf Note}\\
\indent
For the serial and multithreaded environments, the value of {\tt comm} is ignored.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{lis\_vector\_destroy}
\begin{screen}
\verb|C       LIS_INT lis_vector_destroy(LIS_VECTOR v)|\\
\verb|Fortran subroutine lis_vector_destroy(LIS_VECTOR v, LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Destroy vector $v$.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt v] The vector to be destroyed
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt ierr] The return code
\end{namelist}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\subsubsection{lis\_vector\_duplicate}
\begin{screen}
\verb|C       LIS_INT lis_vector_duplicate(void *vin, LIS_VECTOR *vout)|
\verb|Fortran subroutine lis_vector_duplicate(LIS_VECTOR vin, LIS_VECTOR vout,|\\
\verb|         LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Create vector $v_{out}$, which has the same information as $v_{in}$.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt vin] The source vector or matrix
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt vout] The destination vector
\item[\tt ierr] The return code
\end{namelist}
{\bf Note}\\
\indent
The function \verb|lis_vector_duplicate| does not copy the values, 
but allocates only the memory. To copy the values as well, 
the function \verb|lis_vector_copy| must be called after this function.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{lis\_vector\_set\_size}
\begin{screen}
\verb|C       LIS_INT lis_vector_set_size(LIS_VECTOR v, LIS_INT local_n,|\\
\verb|         LIS_INT global_n)|\\
\verb|Fortran subroutine lis_vector_set_size(LIS_VECTOR v, LIS_INTEGER local_n,|\\
\verb|         LIS_INTEGER global_n, LIS_INTEGER ierr)| 
\end{screen}
{\bf Description}\\
\indent
Assign the size of vector $v$.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt v] The vector
\item[\tt local\_n] The size of the partial vector
\item[\tt global\_n] The size of the global vector
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt ierr] The return code
\end{namelist}
{\bf Note}\\
\indent
Either $local\_n$ or $global\_n$ must be provided.

For the serial and multithreaded environments, $local\_n$ 
is equal to $global\_n$. 
Therefore, both \\
\verb|lis_vector_set_size(v,n,0)| and \verb|lis_vector_set_size(v,0,n)| create a vector of size $n$. 

For the multiprocessing environment, \verb|lis_vector_set_size(v,n,0)| 
creates a partial vector of size $n$ on each processing element. On the other hand, 
\verb|lis_vector_set_size(v,0,n)| creates a partial vector of size 
$m_p$ on processing element $p$. The values of $m_p$ are determined by the library. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{lis\_vector\_get\_size}
\begin{screen}
\verb|C       LIS_INT lis_vector_get_size(LIS_VECTOR v, LIS_INT *local_n,|\\
\verb|         LIS_INT *global_n)|\\
\verb|Fortran subroutine lis_vector_get_size(LIS_VECTOR v, LIS_INTEGER local_n,|\\
\verb|         LIS_INTEGER global_n, LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Get the size of vector $v$.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt v] The vector
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt local\_n] The size of the partial vector
\item[\tt global\_n] The size of the global vector
\item[\tt ierr] The return code
\end{namelist}
{\bf Note}\\
\indent
For the serial and multithreaded environments, $local\_n$ is equal to $global\_n$. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{lis\_vector\_get\_range}
\begin{screen}
\verb|C       LIS_INT lis_vector_get_range(LIS_VECTOR v, LIS_INT *is, LIS_INT *ie)|
\verb|Fortran subroutine lis_vector_get_range(LIS_VECTOR v, LIS_INTEGER is,|\\
\verb|         LIS_INTEGER ie, LIS_INTEGER ierr) |
\end{screen}
{\bf Description}\\
\indent
Get the location of the partial vector $v$ in the global vector.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt v] The partial vector
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt is] The location where the partial vector $v$ starts in the global vector
\item[\tt ie] The location where the partial vector $v$ ends in the global vector
\item[\tt ierr] The return code
\end{namelist}
{\bf Note}\\
\indent
For the serial and multithreaded environments, a vector of size $n$ results in $is = 0$ and $ie = n$. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\subsubsection{lis\_vector\_set\_value}
\begin{screen}
\verb|C       LIS_INT lis_vector_set_value(LIS_INT flag, LIS_INT i, LIS_SCALAR value,|
\verb|         LIS_VECTOR v)|\\
\verb|Fortran subroutine lis_vector_set_value(LIS_INTEGER flag, LIS_INTEGER i,|\\
\verb|         LIS_SCALAR value, LIS_VECTOR v, LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Assign the scalar value to the $i$-th row of vector $v$.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt flag] \begin{description}
\item[\tt LIS\_INS\_VALUE]: $v[i] = value$
\item[\tt LIS\_ADD\_VALUE]: $v[i] = v[i] + value$
\end{description}
\item[\tt i] The location where the value is assigned
\item[\tt value] The scalar value to be assigned
\item[\tt v] The vector
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt v] The vector with the scalar value assigned to the $i$-th row
\item[\tt ierr] The return code
\end{namelist}
{\bf Note}\\
\indent
For the multiprocessing environment, the $i$-th row of the global vector must be specified instead 
of the $i$-th row of the partial vector.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{lis\_vector\_get\_value}
\begin{screen}
\verb|C       LIS_INT lis_vector_get_value(LIS_VECTOR v, LIS_INT i, LIS_SCALAR *value)|
\verb|Fortran subroutine lis_vector_get_value(LIS_VECTOR v, LIS_INTEGER i,|\\
\verb|         LIS_SCALAR value, LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Get the scalar value of the $i$-th row of vector $v$.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt i] The location where the value is assigned
\item[\tt v] The source vector
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt value] The value of the $i$-th row
\item[\tt ierr] The return code
\end{namelist}
{\bf Note}\\
\indent
For the multiprocessing environment, the $i$-th row of the global vector must be
specified.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\subsubsection{lis\_vector\_set\_values}
\begin{screen}
\verb|C       LIS_INT lis_vector_set_values(LIS_INT flag, LIS_INT count,|\\
\verb|         LIS_INT index[], LIS_SCALAR value[], LIS_VECTOR v)|\\
\verb|Fortran subroutine lis_vector_set_values(LIS_INTEGER flag, LIS_INTEGER count,|\\
\verb|         LIS_INTEGER index(), LIS_SCALAR value(), LIS_VECTOR v, LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Assign scalar $value[i]$ to the $index[i]$-th row of
vector $v$, where $i=0,1,...,count-1$.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt flag] \begin{description}
\item[\tt LIS\_INS\_VALUE]: $v[index[i]] = value[i]$
\item[\tt LIS\_ADD\_VALUE]: $v[index[i]] = v[index[i]] + value[i]$
\end{description}
\item[\tt count] The number of elements in the array that stores
		the scalar values to be assigned
\item[\tt index] The array that stores the location where the scalar values are assigned
\item[\tt value] The array that stores the scalar values to be assigned
\item[\tt v] The vector
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt v] The vector with scalar $value[i]$ assigned to the $index[i]$-th row
\item[\tt ierr] The return code
\end{namelist}
{\bf Note}\\
\indent
For the multiprocessing environment, the $index[i]$-th row of the global vector must be specified instead 
of the $index[i]$-th row of the partial vector.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\subsubsection{lis\_vector\_get\_values}
\begin{screen}
\verb|C       LIS_INT lis_vector_get_values(LIS_VECTOR v, LIS_INT start, LIS_INT count,|\\
\verb|         LIS_SCALAR value[])|\\
\verb|Fortran subroutine lis_vector_get_values(LIS_VECTOR v, LIS_INTEGER start,|\\
\verb|         LIS_INTEGER count, LIS_SCALAR value(), LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Get scalar $value[i]$ of the $start+i$-th row of vector $v$, 
where $i=0,1,...,count-1$.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt start] The starting location 
\item[\tt count] The number of values to get
\item[\tt v] The source vector
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt value] The array to store the scalar values
\item[\tt ierr] The return code
\end{namelist}
{\bf Note}\\
\indent
For the multiprocessing environment, the $start+i$-th row of the global vector
must be specified.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{lis\_vector\_scatter}
\begin{screen}
\verb|C       LIS_INT lis_vector_scatter(LIS_SCALAR value[], LIS_VECTOR v)|\\
\verb|Fortran subroutine lis_vector_scatter(LIS_SCALAR value(), LIS_VECTOR v,|\\
\verb|         LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Assign scalar $value[i]$ to the $i$-th row of vector $v$, 
where $i=0,1,..., global\_n-1$.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt value] The array that stores the scalar values to be assigned
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt v] The vector
\item[\tt ierr] The return code
\end{namelist}
{\bf Note}\\
\indent

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\subsubsection{lis\_vector\_gather}
\begin{screen}
\verb|C       LIS_INT lis_vector_gather(LIS_VECTOR v, LIS_SCALAR value[])|\\
\verb|Fortran subroutine lis_vector_gather(LIS_VECTOR v, LIS_SCALAR value(),|\\
\verb|         LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Get scalar $value[i]$ of the $i$-th row of vector $v$, 
where $i=0,1,..., global\_n-1$.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt v] The source vector
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt value] The array that stores the scalar values
\item[\tt ierr] The return code
\end{namelist}
{\bf Note}\\
\indent

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{lis\_vector\_copy}
\begin{screen}
\verb|C       LIS_INT lis_vector_copy(LIS_VECTOR x, LIS_VECTOR y)|\\
\verb|Fortran subroutine lis_vector_copy(LIS_VECTOR x, LIS_VECTOR y, LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Copy the values of the vector elements.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt x] The source vector
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt y] The destination vector
\item[\tt ierr] The return code
\end{namelist}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{lis\_vector\_set\_all}
\begin{screen}
\verb|C       LIS_INT lis_vector_set_all(LIS_SCALAR value, LIS_VECTOR v)|
\verb|Fortran subroutine lis_vector_set_all(LIS_SCALAR value, LIS_VECTOR v,|\\
\verb|         LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Assign the scalar value to the elements of vector $v$.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt value] The scalar value to be assigned
\item[\tt v] The vector
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt v] The vector with the value assigned to the elements
\item[\tt ierr] The return code
\end{namelist}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\subsection{Operating Matrix Elements}
Assume that the size of matrix $A$ is $global\_n$ $\times$ $global\_n$
and that the size of each partial matrix stored on $nprocs$ 
processing elements is $local\_n$ $\times$ $global\_n$. Here, $global\_n$
and $local\_n$ are called the number of rows of the global matrix
and the number of rows of the partial matrix, respectively. 
\subsubsection{lis\_matrix\_create}
\begin{screen}
\verb|C       LIS_INT lis_matrix_create(LIS_Comm comm, LIS_MATRIX *A)|
\verb|Fortran subroutine lis_matrix_create(LIS_Comm comm, LIS_MATRIX A, LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Create matrix $A$.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt LIS\_Comm] The MPI communicator
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt A] The matrix
\item[\tt ierr] The return code
\end{namelist}
{\bf Note}\\
\indent
For the serial and multithreaded environments, the value of {\tt comm} is ignored. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{lis\_matrix\_destroy}
\begin{screen}
\verb|C       LIS_INT lis_matrix_destroy(LIS_MATRIX A)|\\
\verb|Fortran subroutine lis_matrix_destroy(LIS_MATRIX A, LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Destroy matrix $A$.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt A] The matrix to be destroyed
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt ierr] The return code
\end{namelist}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\subsubsection{lis\_matrix\_duplicate}
\begin{screen}
\verb|C       LIS_INT lis_matrix_duplicate(LIS_MATRIX Ain, LIS_MATRIX *Aout)|
\verb|Fortran subroutine lis_matrix_duplicate(LIS_MATRIX Ain, LIS_MATRIX Aout,|\\
\verb|         LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Create matrix $A_{out}$ which has the same information as $A_{in}$.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt Ain] The source matrix
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt Aout] The destination matrix
\item[\tt ierr] The return code
\end{namelist}
{\bf Note}\\
\indent
The function \verb|lis_matrix_duplicate| does not copy the values of 
the elements of the matrix, but allocates only the memory. 
To copy the values of the elements as well, the function
\verb|lis_matrix_copy| must be called after this function.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{lis\_matrix\_malloc}
\begin{screen}
\verb|C       LIS_INT lis_matrix_malloc(LIS_MATRIX A, LIS_INT nnz_row, LIS_INT nnz[])|
\verb|Fortran subroutine lis_matrix_malloc(LIS_MATRIX A, LIS_INTEGER nnz_row,|\\
\verb|         LIS_INTEGER nnz[], LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Allocate the memory for matrix $A$.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt A] The matrix
\item[\tt nnz\_row] The average number of nonzero elements
\item[\tt nnz] The array of numbers of nonzero elements in each row
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt ierr] The return code
\end{namelist}
{\bf Note}\\
\indent
Either \verb|nnz_row| or \verb|nnz| must be provided.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\subsubsection{lis\_matrix\_set\_value}
\begin{screen}
\verb|C       LIS_INT lis_matrix_set_value(LIS_INT flag, LIS_INT i, LIS_INT j,|\\
\verb|         LIS_SCALAR value, LIS_MATRIX A)|\\
\verb|Fortran subroutine lis_matrix_set_value(LIS_INTEGER flag, LIS_INTEGER i,|\\
\verb|         LIS_INTEGER j, LIS_SCALAR value, LIS_MATRIX A, LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Assign the scalar value to the $(i, j)$-th element of matrix $A$.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt flag] \begin{description}
\item[\tt LIS\_INS\_VALUE]: $A[i,j] = value$
\item[\tt LIS\_ADD\_VALUE]: $A[i,j] = A[i,j] + value$
\end{description}
\item[\tt i] The row number of the matrix
\item[\tt j] The column number of the matrix
\item[\tt value] The value to be assigned
\item[\tt A] The matrix
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt A] The matrix 
\item[\tt ierr] The return code
\end{namelist}
\noindent
{\bf Note}\\
\indent
For the multiprocessing environment, the $i$-th row and the $j$-th column of the global matrix must 
be specified. 

The function \verb|lis_matrix_set_value| stores the assigned value in a temporary internal format. 
Therefore, after \verb|lis_matrix_set_value| is called, 
the function \verb|lis_matrix_assemble| must be called. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{lis\_matrix\_assemble}
\begin{screen}
\verb|C       LIS_INT lis_matrix_assemble(LIS_MATRIX A)|\\
\verb|Fortran subroutine lis_matrix_assemble(LIS_MATRIX A, LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Assemble matrix $A$ into the specified storage format.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt A] The matrix
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt A] The matrix assembled into the specified storage format
\item[\tt ierr] The return code
\end{namelist}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{lis\_matrix\_set\_size}
\begin{screen}
\verb|C       LIS_INT lis_matrix_set_size(LIS_MATRIX A, LIS_INT local_n,|\\
\verb|         LIS_INT global_n)|\\
\verb|Fortran subroutine lis_matrix_set_size(LIS_MATRIX A, LIS_INTEGER local_n,|\\
\verb|         LIS_INTEGER global_n, LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Assign the size of matrix $A$.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt A] The matrix
\item[\tt local\_n] The number of rows of the partial matrix
\item[\tt global\_n] The number of rows of the global matrix
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt ierr] The return code
\end{namelist}
{\bf Note}\\
\indent
Either $local\_n$ or $global\_n$ must be provided. 

For the serial and multithreaded environments, $local\_n$ is
equal to $global\_n$. 
Therefore, both \\
\verb|lis_matrix_set_size(A,n,0)| and \verb|lis_matrix_set_size(A,0,n)|
create a matrix of size $n \times n$.

For the multiprocessing environment, \verb|lis_matrix_set_size(A,n,0)| creates
a partial matrix of size $n \times N$ on each processing element, 
where $N$ is the total sum of $n$. 
On the other hand, \verb|lis_matrix_set_size(A,0,n)| creates 
a partial matrix of size $m_p \times n$ on processing element $p$.
The values of $m_p$ are determined by the library.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{lis\_matrix\_get\_size}
\begin{screen}
\verb|C       LIS_INT lis_matrix_get_size(LIS_MATRIX A, LIS_INT *local_n,|
\verb|         LIS_INT *global_n)|\\
\verb|Fortran subroutine lis_matrix_get_size(LIS_MATRIX A, LIS_INTEGER local_n,|\\
\verb|         LIS_INTEGER global_n, LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Get the size of matrix $A$.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt A] The matrix
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt local\_n] The number of rows of the partial matrix
\item[\tt global\_n] The number of rows of the global matrix
\item[\tt ierr] The return code
\end{namelist}
{\bf Note}\\
\indent
For the serial and multithreaded environments, $local\_n$ is
equal to $global\_n$. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{lis\_matrix\_get\_range}
\begin{screen}
\verb|C       LIS_INT lis_matrix_get_range(LIS_MATRIX A, LIS_INT *is, LIS_INT *ie)|
\verb|Fortran subroutine lis_matrix_get_range(LIS_MATRIX A, LIS_INTEGER is,|\\
\verb|         LIS_INTEGER ie, LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Get the location of the partial matrix $A$ in the global matrix.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt A] The partial matrix
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt is] The location where the partial matrix $A$ starts in the global matrix
\item[\tt ie] The location where the partial matrix $A$ ends in the global matrix
\item[\tt ierr] The return code
\end{namelist}
{\bf Note}\\
\indent
For the serial and multithreaded environments, a matrix of $n \times n$ results in $is = 0$ and $ie = n$.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\subsubsection{lis\_matrix\_set\_type}
\begin{screen}
\verb|C       LIS_INT lis_matrix_set_type(LIS_MATRIX A, LIS_INT matrix_type)|
\verb|Fortran subroutine lis_matrix_set_type(LIS_MATRIX A, LIS_INTEGER matrix_type,|\\
\verb|         LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Assign the storage format.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt A] The matrix
\item[\tt matrix\_type] The storage format
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt ierr] The return code
\end{namelist}
\noindent
{\bf Note}\\
\indent
\verb+matrix_type+ of $A$ is \verb+LIS_MATRIX_CSR+ when the matrix is created. 
The table below shows the available storage formats for \verb+matrix_type+.
\\ \\
\begin{minipage}[t]{\textwidth}
\begin{center}
\begin{tabular}{lll}\hline\hline
Storage format  & & \verb|matrix_type| \\ \hline
Compressed Sparse Row & (CSR) & \verb={LIS_MATRIX_CSR|1}= \\
Compressed Sparse Column & (CSC) & \verb={LIS_MATRIX_CSC|2}= \\
Modified Compressed Sparse Row & (MSR) & \verb={LIS_MATRIX_MSR|3}= \\
Diagonal &(DIA) & \verb={LIS_MATRIX_DIA|4}= \\
Ellpack-Itpack Generalized Diagonal &(ELL) & \verb={LIS_MATRIX_ELL|5}= \\
Jagged Diagonal &(JAD) & \verb={LIS_MATRIX_JAD|6}= \\
Block Sparse Row & (BSR) & \verb={LIS_MATRIX_BSR|7}= \\
Block Sparse Column &(BSC) & \verb={LIS_MATRIX_BSC|8}= \\
Variable Block Row &(VBR) & \verb={LIS_MATRIX_VBR|9}= \\
Coordinate & (COO) & \verb={LIS_MATRIX_COO|10}= \\
Dense &	(DNS) & \verb={LIS_MATRIX_DNS|11}= \\
\hline         
\end{tabular}
\end{center}
\end{minipage}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{lis\_matrix\_get\_type}
\begin{screen}
\verb|C       LIS_INT lis_matrix_get_type(LIS_MATRIX A, LIS_INT *matrix_type)|
\verb|Fortran subroutine lis_matrix_get_type(LIS_MATRIX A, LIS_INTEGER matrix_type,|\\
\verb|         LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Get the storage format.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt A] The matrix
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt matrix\_type] The storage format
\item[\tt ierr] The return code
\end{namelist}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{lis\_matrix\_set\_blocksize}
\begin{screen}
\verb|C       LIS_INT lis_matrix_set_blocksize(LIS_MATRIX A, LIS_INT bnr, LIS_INT bnc,|\\
\verb|         LIS_INT row[], LIS_INT col[])|\\
\verb|Fortran subroutine lis_matrix_set_blocksize(LIS_MATRIX A, LIS_INTEGER bnr,|\\
\verb|         LIS_INTEGER bnc, LIS_INTEGER row[], LIS_INTEGER col[], LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Assign the block size of the BSR, BSC, and VBR formats.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt A] The matrix
\item[\tt bnr] The row block size of the BSR (BSC) format or the number of 
	     row blocks of the VBR format
\item[\tt bnc] The olumn block size of the BSR (BSC) format or the
	     number of column blocks of the VBR format
\item[\tt row] The array of the row division information about the VBR format
\item[\tt col] The array of the column division information about the
	     VBR format
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt ierr] The return code
\end{namelist}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{lis\_matrix\_convert}
\begin{screen}
\verb|C       LIS_INT lis_matrix_convert(LIS_MATRIX Ain, LIS_MATRIX Aout)|\\
\verb|Fortran subroutine lis_matrix_convert(LIS_MATRIX Ain, LIS_MATRIX Aout,|\\
\verb|         LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Convert matrix $A_{in}$ into $A_{out}$ of the format specified by {\tt lis\_matrix\_set\_type}.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt Ain] The source matrix
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt Aout] The destination matrix
\item[\tt ierr] The return code
\end{namelist}
{\bf Note}\\
\indent
The storage format of $A_{out}$ is set by \verb|lis_matrix_set_type|. 
The block size of the BSR, BSC, and VBR formats is set by \verb|lis_matrix_set_blocksize|. 

The conversions indicated by 1 in the table below are performed directly, 
and the others are performed via the indicated formats. 
The conversions with no indication are performed via the CSR format.
\vspace*{5mm}

\begin{tabular}{|c|c|c|c|c|c|c|c|c|c|c|c|}
\hline
Src \textbackslash Dst & CSR & CSC & MSR & DIA & ELL & JAD & BSR & BSC & VBR & COO & DNS \\ \hline
CSR    &     &  1  &  1  &  1  &  1  &  1  &  1  &  CSC  &  1  &  1  &  1  \\ \hline
COO    &  1  &  1  &  1  & CSR  & CSR & CSR & CSR & CSC & CSR &     & CSR   \\ \hline
\end{tabular}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{lis\_matrix\_copy}
\begin{screen}
\verb|C       LIS_INT lis_matrix_copy(LIS_MATRIX Ain, LIS_MATRIX Aout)|
\verb|Fortran subroutine lis_matrix_copy(LIS_MATRIX Ain, LIS_MATRIX Aout,|\\
\verb|         LIS_INTEGER ierr) |
\end{screen}
{\bf Description}\\
\indent
Copy the values of the matrix elements.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt Ain] The source matrix
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt Aout] The destination matrix
\item[\tt ierr] The return code
\end{namelist}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{lis\_matrix\_get\_diagonal}
\begin{screen}
\verb|C       LIS_INT lis_matrix_get_diagonal(LIS_MATRIX A, LIS_VECTOR d)|\\
\verb|Fortran subroutine lis_matrix_get_diagonal(LIS_MATRIX A, LIS_VECTOR d,|\\
\verb|         LIS_INTEGER ierr) |
\end{screen}
{\bf Description}\\
\indent
Store the diagonal elements of matrix $A$ to vector $d$.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt A] The matrix
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt d] The vector that stores the diagonal elements of the matrix
\item[\tt ierr] The return code
\end{namelist}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\subsubsection{lis\_matrix\_set\_csr}
\begin{screen}
\verb|C       LIS_INT lis_matrix_set_csr(LIS_INT nnz, LIS_INT ptr[], LIS_INT index[],|\\
\verb|         LIS_SCALAR value[], LIS_MATRIX A)|\\
\verb|Fortran subroutine lis_matrix_set_csr(LIS_INTEGER nnz, LIS_INTEGER row(),|\\
\verb|         LIS_INTEGER index(), LIS_SCALAR value(), LIS_MATRIX A, LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Associate the arrays in the CSR format with matrix $A$.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt nnz] The number of nonzero elements
\item[\tt ptr, index, value] The arrays in the CSR format
\item[\tt A] The matrix
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt A] The matrix associated with the arrays
\end{namelist}
\noindent
{\bf Note}\\
\indent
After \verb|lis_matrix_set_csr| is called, 
the function \verb|lis_matrix_assemble| must be called. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{lis\_matrix\_set\_csc}
\begin{screen}
\verb|C       LIS_INT lis_matrix_set_csc(LIS_INT nnz, LIS_INT ptr[], LIS_INT index[],|\\
\verb|         LIS_SCALAR value[], LIS_MATRIX A)|\\
\verb|Fortran subroutine lis_matrix_set_csc(LIS_INTEGER nnz, LIS_INTEGER row(),|\\
\verb|         LIS_INTEGER index(), LIS_SCALAR value(), LIS_MATRIX A, LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Associate the arrays in the CSC format with matrix $A$.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt nnz] The number of nonzero elements
\item[\tt ptr, index, value] The arrays in the CSC format
\item[\tt A] The matrix
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt A] The matrix associated with the arrays 
\end{namelist}
\noindent
{\bf Note}\\
\indent
After \verb|lis_matrix_set_csc| is called, 
the function \verb|lis_matrix_assemble| must be called. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\subsubsection{lis\_matrix\_set\_msr}
\begin{screen}
\verb|C       LIS_INT lis_matrix_set_msr(LIS_INT nnz, LIS_INT ndz, LIS_INT index[],|\\
\verb|         LIS_SCALAR value[], LIS_MATRIX A)| \\
\verb|Fortran subroutine lis_matrix_set_msr(LIS_INTEGER nnz, LIS_INTEGER ndz,|\\
\verb|         LIS_INTEGER index(), LIS_SCALAR value(), LIS_MATRIX A, LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Associate the arrays in the MSR format with matrix $A$.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt nnz] The number of nonzero elements
\item[\tt ndz] The number of nonzero elements in the diagonal
\item[\tt index, value] The arrays in the MSR format
\item[\tt A] The matrix
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt A] The matrix associated with the arrays
\end{namelist}
\noindent
{\bf Note}\\
\indent
After \verb|lis_matrix_set_msr| is called, 
the function \verb|lis_matrix_assemble| must be called. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{lis\_matrix\_set\_dia}
\begin{screen}
\verb|C       LIS_INT lis_matrix_set_dia(LIS_INT nnd, LIS_INT index[],|\\
\verb|         LIS_SCALAR value[], LIS_MATRIX A)|\\
\verb|Fortran subroutine lis_matrix_set_dia(LIS_INTEGER nnd, LIS_INTEGER index(), |\\
\verb|         LIS_SCALAR value(), LIS_MATRIX A, LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Associate the arrays in the DIA format with matrix $A$.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt nnd] The number of nonzero diagonal elements
\item[\tt index, value] The arrays in the DIA format
\item[\tt A] The matrix
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt A] The matrix associated with the arrays 
\end{namelist}
\noindent
{\bf Note}\\
\indent
After \verb|lis_matrix_set_dia| is called, 
the function \verb|lis_matrix_assemble| must be called. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\subsubsection{lis\_matrix\_set\_ell}
\begin{screen}
\verb|C       LIS_INT lis_matrix_set_ell(LIS_INT maxnzr, LIS_INT index[],|\\
\verb|         LIS_SCALAR value[], LIS_MATRIX A)|\\
\verb|Fortran subroutine lis_matrix_set_ell(LIS_INTEGER maxnzr,|\\
\verb|         LIS_INTEGER index(), LIS_SCALAR value(), LIS_MATRIX A,|\\
\verb|         LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Associate the arrays in the ELL format with matrix $A$. 
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt maxnzr] The maximum number of nonzero elements in each row
\item[\tt index, value] The arrays in the ELL format
\item[\tt A] The matrix
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt A] The matrix associated with the arrays
\end{namelist}
\noindent
{\bf Note}\\
\indent
After \verb|lis_matrix_set_ell| is called, 
the function \verb|lis_matrix_assemble| must be called. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{lis\_matrix\_set\_jad}
\begin{screen}
\verb|C       LIS_INT lis_matrix_set_jad(LIS_INT nnz, LIS_INT maxnzr, LIS_INT perm[],|\\
\verb|         LIS_INT ptr[], LIS_INT index[], LIS_SCALAR value[], LIS_MATRIX A)|\\
\verb|Fortran subroutine lis_matrix_set_jad(LIS_INTEGER nnz, LIS_INTEGER maxnzr,|\\
\verb|         LIS_INTEGER ptr(), LIS_INTEGER index(), LIS_SCALAR value(),|\\
\verb|         LIS_MATRIX A, LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Associate the arrays in the JAD format with matrix $A$.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt nnz] The number of nonzero elements
\item[\tt maxnzr] The maximum number of nonzero elements in each row
\item[\tt perm, ptr, index, value] The arrays in the JAD format
\item[\tt A] The matrix
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt A] The matrix associated with the arrays
\end{namelist}
\noindent
{\bf Note}\\
\indent
After \verb|lis_matrix_set_jad| is called, 
the function \verb|lis_matrix_assemble| must be called. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{lis\_matrix\_set\_bsr}
\begin{screen}
\verb|C       LIS_INT lis_matrix_set_bsr(LIS_INT bnr, LIS_INT bnc, LIS_INT bnnz,|\\
\verb|         LIS_INT bptr[], LIS_INT bindex[], LIS_SCALAR value[], LIS_MATRIX A)|\\
\verb|Fortran subroutine lis_matrix_set_bsr(LIS_INTEGER bnr, LIS_INTEGER bnc,|\\
\verb|         LIS_INTEGER bnnz, LIS_INTEGER bptr(), LIS_INTEGER bindex(),|\\
\verb|         LIS_SCALAR value(), LIS_MATRIX A, LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Associate the arrays in the BSR format with matrix $A$.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt bnr] The row block size
\item[\tt bnc] The column block size
\item[\tt bnnz] The number of nonzero blocks
\item[\tt bptr, bindex, value] The arrays in the BSR format
\item[\tt A] The matrix
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt A] The matrix associated with the arrays
\end{namelist}
\noindent
{\bf Note}\\
\indent
After \verb|lis_matrix_set_bsr| is called, 
the function \verb|lis_matrix_assemble| must be called. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{lis\_matrix\_set\_bsc}
\begin{screen}
\verb|C       LIS_INT lis_matrix_set_bsc(LIS_INT bnr, LIS_INT bnc, LIS_INT bnnz,|\\
\verb|         LIS_INT bptr[], LIS_INT bindex[], LIS_SCALAR value[], LIS_MATRIX A)|\\
\verb|Fortran subroutine lis_matrix_set_bsc(LIS_INTEGER bnr, LIS_INTEGER bnc,|\\
\verb|         LIS_INTEGER bnnz, LIS_INTEGER bptr(), LIS_INTEGER bindex(),|\\
\verb|         LIS_SCALAR value(), LIS_MATRIX A, LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Associate the arrays in the BSC format with matrix $A$.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt bnr] The row block size
\item[\tt bnc] The column block size
\item[\tt bnnz] The number of nonzero blocks
\item[\tt bptr, bindex, value] The arrays in the BSC format
\item[\tt A] The matrix
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt A] The matrix associated with the arrays
\end{namelist}
\noindent
{\bf Note}\\
\indent
After \verb|lis_matrix_set_bsc| is called, 
the function \verb|lis_matrix_assemble| must be called. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\subsubsection{lis\_matrix\_set\_vbr}
\begin{screen}
\verb|C       LIS_INT lis_matrix_set_vbr(LIS_INT nnz, LIS_INT nr, LIS_INT nc,|\\
\verb|         LIS_INT bnnz, LIS_INT row[], LIS_INT col[], LIS_INT ptr[],|\\
\verb|         LIS_INT bptr[], LIS_INT bindex[], LIS_SCALAR value[],|\\
\verb|         LIS_MATRIX A)|\\
\verb|Fortran subroutine lis_matrix_set_vbr(LIS_INTEGER nnz, LIS_INTEGER nr,|\\
\verb|         LIS_INTEGER nc, LIS_INTEGER bnnz, LIS_INTEGER row(),|\\
\verb|         LIS_INTEGER col(), LIS_INTEGER ptr(), LIS_INTEGER bptr(),|\\
\verb|         LIS_INTEGER bindex(), LIS_SCALAR value(), LIS_MATRIX A,|\\
\verb|         LIS_INTEGER ierr) |
\end{screen}
{\bf Description}\\
\indent
Associate the arrays in the VBR format with matrix $A$.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt nnz] The number of nonzero elements
\item[\tt nr] The number of row blocks
\item[\tt nc] The number of column blocks
\item[\tt bnnz] The number of nonzero blocks
\item[\tt row, col, ptr, bptr, bindex, value] The arrays in the VBR format
\item[\tt A] The matrix
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt A] The matrix associated with the arrays
\end{namelist}
\noindent
{\bf Note}\\
\indent
After \verb|lis_matrix_set_vbr| is called, 
the function \verb|lis_matrix_assemble| must be called. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{lis\_matrix\_set\_coo}
\begin{screen}
\verb|C       LIS_INT lis_matrix_set_coo(LIS_INT nnz, LIS_INT row[], LIS_INT col[],|\\
\verb|         LIS_SCALAR value[], LIS_MATRIX A)|\\
\verb|Fortran subroutine lis_matrix_set_coo(LIS_INTEGER nnz, LIS_INTEGER row(),|\\
\verb|         LIS_INTEGER col(), LIS_SCALAR value(), LIS_MATRIX A, LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Associate the arrays in the COO format with matrix $A$.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt nnz] The number of nonzero elements
\item[\tt row, col, value] The arrays in the COO format
\item[\tt A] The matrix
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt A] The matrix associated with the arrays
\end{namelist}
\noindent
{\bf Note}\\
\indent
After \verb|lis_matrix_set_coo| is called, 
the function \verb|lis_matrix_assemble| must be called. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{lis\_matrix\_set\_dns}
\begin{screen}
\verb|C       LIS_INT lis_matrix_set_dns(LIS_SCALAR value[], LIS_MATRIX A)|\\
\verb|Fortran subroutine lis_matrix_set_dns(LIS_SCALAR value(), LIS_MATRIX A,|\\
\verb|        LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Associate the array in the DNS format with matrix $A$.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt value] The array in the DNS format
\item[\tt A] The matrix
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt A] The matrix associated with the array
\end{namelist}
\noindent
{\bf Note}\\
\indent
After \verb|lis_matrix_set_dns| is called, 
the function \verb|lis_matrix_assemble| must be called. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\subsection{Operating Vectors and Matrices}
\subsubsection{lis\_vector\_scale}
\begin{screen}
\verb|C       LIS_INT lis_vector_scale(LIS_SCALAR alpha, LIS_VECTOR x)|
\verb|Fortran subroutine lis_vector_scale(LIS_SCALAR alpha, LIS_VECTOR x,|\\
\verb|         LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Multiply vector $x$ by scalar $\alpha$.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt alpha] The scalar value
\item[\tt x] The vector
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt x] $\alpha x$ (vector $x$ is overwritten)
\item[\tt ierr] The return code
\end{namelist}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{lis\_vector\_dot}
\begin{screen}
\verb|C       LIS_INT lis_vector_dot(LIS_VECTOR x, LIS_VECTOR y, LIS_SCALAR *val)|
\verb|Fortran subroutine lis_vector_dot(LIS_VECTOR x, LIS_VECTOR y, LIS_SCALAR val,|\\
\verb|         LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Calculate the inner product $x^{T}y$.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt x] The vector
\item[\tt y] The vector
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt val] The inner product
\item[\tt ierr] The return code
\end{namelist}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\subsubsection{lis\_vector\_nrm1}
\begin{screen}
\verb|C       LIS_INT lis_vector_nrm1(LIS_VECTOR x, LIS_SCALAR *val)|\\
\verb|Fortran subroutine lis_vector_nrm1(LIS_VECTOR x, LIS_SCALAR val, LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Calculate the 1-norm of vector $x$.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt x] The vector
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt val] The 1-norm of the vector
\item[\tt ierr] The return code
\end{namelist}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{lis\_vector\_nrm2}
\begin{screen}
\verb|C       LIS_INT lis_vector_nrm2(LIS_VECTOR x, LIS_SCALAR *val)|\\
\verb|Fortran subroutine lis_vector_nrm2(LIS_VECTOR x, LIS_SCALAR val, LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Calculate the 2-norm of vector $x$.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt x] The vector
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt val] The 2-norm of the vector
\item[\tt ierr] The return code
\end{namelist}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{lis\_vector\_nrmi}
\begin{screen}
\verb|C       LIS_INT lis_vector_nrmi(LIS_VECTOR x, LIS_SCALAR *val)|\\
\verb|Fortran subroutine lis_vector_nrmi(LIS_VECTOR x, LIS_SCALAR val, LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Calculate the infinity norm of vector $x$.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt x] The vector
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt val] The infinity norm of the vector
\item[\tt ierr] The return code
\end{namelist}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\subsubsection{lis\_vector\_axpy}
\begin{screen}
\verb|C       LIS_INT lis_vector_axpy(LIS_SCALAR alpha, LIS_VECTOR x, LIS_VECTOR y)|
\verb|Fortran subroutine lis_vector_axpy(LIS_SCALAR alpha, LIS_VECTOR x, LIS_VECTOR y,|\\
\verb|         LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Calculate the sum of the vectors $y = \alpha x + y$.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt alpha] The scalar value
\item[\tt x, y] The vectors
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt y] $\alpha x + y$ (vector $y$ is overwritten)
\item[\tt ierr] The return code
\end{namelist}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{lis\_vector\_xpay}
\begin{screen}
\verb|C       LIS_INT lis_vector_xpay(LIS_VECTOR x, LIS_SCALAR alpha, LIS_VECTOR y)|
\verb|Fortran subroutine lis_vector_xpay(LIS_VECTOR x, LIS_SCALAR alpha, LIS_VECTOR y,|\\
\verb|         LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Calculate the sum of the vectors $y = x + \alpha y$.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt alpha] The scalar value
\item[\tt x, y] The vectors
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt y] $x + \alpha y$ (vector $y$ is overwritten)
\item[\tt ierr] The return code
\end{namelist}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\subsubsection{lis\_vector\_axpyz}
\begin{screen}
\verb|C       LIS_INT lis_vector_axpyz(LIS_SCALAR alpha, LIS_VECTOR x, LIS_VECTOR y,|\\
\verb|         LIS_VECTOR z)|\\
\verb|Fortran subroutine lis_vector_axpyz(LIS_SCALAR alpha, LIS_VECTOR x, LIS_VECTOR y,|\\
\verb|         LIS_VECTOR z, LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Calculate the sum of the vectors $z = \alpha x + y$.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt alpha] The scalar value
\item[\tt x, y] The vectors
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt z] $x + \alpha y$
\item[\tt ierr] The return code
\end{namelist}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{lis\_matrix\_scaling}
\begin{screen}
\verb|C       LIS_INT lis_matrix_scaling(LIS_MATRIX A, LIS_VECTOR b, LIS_VECTOR d,|\\
\verb|         LIS_INT action)|\\
\verb|Fortran subroutine lis_matrix_scaling(LIS_MATRIX A, LIS_VECTOR b,|\\
\verb|         LIS_VECTOR d, LIS_INTEGER action, LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Scale matrix $A$.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt A] The matrix
\item[\tt b] The vector
\item[\tt action] \begin{description}
\item[\tt LIS\_SCALE\_JACOBI]: Jacobi scaling $D^{-1}Ax=D^{-1}b$, where $D$
	   represents the diagonal of $A=(a_{ij})$
\item[\tt LIS\_SCALE\_SYMM\_DIAG]: Diagonal scaling
	   $D^{-1/2}AD^{-1/2}x=D^{-1/2}b$, where $D^{-1/2}$ represents
	   a diagonal matrix with $1/\sqrt{a_{ii}}$ as the diagonal
\end{description}
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt d]  The vector that stores the diagonal elements of $D^{-1}$ or $D^{-1/2}$
\item[\tt ierr] The return code
\end{namelist}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\subsubsection{lis\_matvec}
\begin{screen}
\verb|C       LIS_INT lis_matvec(LIS_MATRIX A, LIS_VECTOR x, LIS_VECTOR y)|\\
\verb|Fortran subroutine lis_matvec(LIS_MATRIX A, LIS_VECTOR x, LIS_VECTOR y,|\\
\verb|         LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Calculate the matrix vector product $y=Ax$.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt A] The matrix
\item[\tt x] The vector
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt y] $Ax$
\item[\tt ierr] The return code
\end{namelist}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{lis\_matvect}
\begin{screen}
\verb|C       LIS_INT lis_matvect(LIS_MATRIX A, LIS_VECTOR x, LIS_VECTOR y)|\\
\verb|Fortran subroutine lis_matvect(LIS_MATRIX A, LIS_VECTOR x, LIS_VECTOR y,|\\
\verb|         LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Calculate the transposed matrix vector product $y=A^{T}x$.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt A] The matrix
\item[\tt x] The vector
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt y] $A^{T}x$
\item[\tt ierr] The return code
\end{namelist}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\subsection{Solving Linear Equations}
\subsubsection{lis\_solver\_create}
\begin{screen}
\verb|C       LIS_INT lis_solver_create(LIS_SOLVER *solver)|\\
\verb|Fortran subroutine lis_solver_create(LIS_SOLVER solver, LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Create the solver.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[None]
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt solver] The solver
\item[\tt ierr] The return code
\end{namelist}
{\bf Note}\\
\indent
\verb|solver| has the information on the solver, the preconditioner, etc.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{lis\_solver\_destroy}
\begin{screen}
\verb|C       LIS_INT lis_solver_destroy(LIS_SOLVER solver)|\\
\verb|Fortran subroutine lis_solver_destroy(LIS_SOLVER solver, LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Destroy the solver.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt solver] The solver to be destroyed
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt ierr] The return code
\end{namelist}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\subsubsection{lis\_solver\_set\_option}
  \label{sec:setoptions}
\begin{screen}
\verb|C       LIS_INT lis_solver_set_option(char *text, LIS_SOLVER solver)|
\verb|Fortran subroutine lis_solver_set_option(character text, LIS_SOLVER solver,|\\
\verb|         LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Set the options for the solver.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt text] The command line options
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt solver] The solver
\item[\tt ierr] The return code
\end{namelist}
{\bf Note}\\
\indent
The table below shows the available command line options, 
where \verb=-i {cg|1}= means \verb=-i cg= or \verb=-i 1= and \verb=-maxiter [1000]= indicates 
that \verb=-maxiter= defaults to $1,000$.
\\
\\
\begin{minipage}[t]{\textwidth}
\begin{center}
{\bf Options for Linear Solvers} (Default: \verb=-i bicg=) \\
\begin{tabular}{l|lll}\hline\hline
 Solver      & Option              &  Auxiliary Options  & \\ \hline
 CG          & \verb=-i {cg|1}=         &    \\ 
 BiCG        & \verb=-i {bicg|2}=       &    \\
 CGS         & \verb=-i {cgs|3}=        &    \\
 BiCGSTAB    & \verb=-i {bicgstab|4}=   &    \\
 BiCGSTAB(l) & \verb=-i {bicgstabl|5}=  & \verb=-ell [2]=      & The degree $l$ \\
 GPBiCG      & \verb=-i {gpbicg|6}=     &    \\
 TFQMR       & \verb=-i {tfqmr|7}=      &    \\
 Orthomin(m) & \verb=-i {orthomin|8}=   & \verb=-restart [40]= & The restart
 value $m$  \\
 GMRES(m)    & \verb=-i {gmres|9}=      & \verb=-restart [40]= & The restart value $m$  \\ 
 Jacobi      & \verb=-i {jacobi|10}=    &    \\
 Gauss-Seidel& \verb=-i {gs|11}=        &    \\
 SOR         & \verb=-i {sor|12}=       & \verb=-omega [1.9]=  & The relaxation coefficient $\omega$ ($0<\omega<2$) \\
 BiCGSafe    & \verb=-i {bicgsafe|13}=     &    \\
 CR          & \verb=-i {cr|14}=        &    \\ 
 BiCR        & \verb=-i {bicr|15}=      &    \\
 CRS         & \verb=-i {crs|16}=       &    \\
 BiCRSTAB    & \verb=-i {bicrstab|17}=  &    \\
 GPBiCR      & \verb=-i {gpbicr|18}=    &    \\
 BiCRSafe    & \verb=-i {bicrsafe|19}=  &    \\
 FGMRES(m)   & \verb=-i {fgmres|20}=    & \verb=-restart [40]= & The restart value $m$   \\ 
 IDR(s)      & \verb=-i {idrs|21}=      & \verb=-irestart [2]= & The restart
 value $s$  \\ 
 MINRES      & \verb=-i {minres|22}=    &    \\
\hline         
\end{tabular}
\end{center}
\end{minipage}
\\ \\
\begin{minipage}[t]{\textwidth}
\begin{center}
{\bf Options for Preconditioners} (Default: \verb=-p none=)\\
\begin{tabular}{l|lll}\hline\hline
Preconditioner   & Option           & Auxiliary Options \\ \hline
None     & \verb=-p {none|0}=    &   \\
Jacobi   & \verb=-p {jacobi|1}=  &     \\
ILU(k)   & \verb=-p {ilu|2}=     & \verb=-ilu_fill [0]=    & The fill level $k$ \\
SSOR     & \verb=-p {ssor|3}=    & \verb=-ssor_w [1.0]=    & The relaxation coefficient $\omega$ ($0<\omega<2$) \\
Hybrid   & \verb=-p {hybrid|4}=  & \verb=-hybrid_i [sor]=  & The linear solver\\
         &                       & \verb=-hybrid_maxiter [25]= & The
 maximum number of iterations \\
         &                       & \verb=-hybrid_tol [1.0e-3]= & The convergence criterion \\
         &                       & \verb=-hybrid_w [1.5]=      & The
 relaxation coefficient $\omega$ of the SOR \\
         &                       &                             & ($0<\omega<2$) \\
         &                       & \verb=-hybrid_ell [2]=      & The
 degree $l$ of the BiCGSTAB(l) \\
         &                       & \verb=-hybrid_restart [40]= & The
 restart values of the GMRES \\
         &                       &                             & and
 Orthomin \\
I+S      & \verb=-p {is|5}=      & \verb=-is_alpha [1.0]=  &  The
 parameter $\alpha$ of $I+\alpha S^{(m)}$ \\
         &                       & \verb=-is_m [3]=        & The
 parameter $m$ of $I+\alpha S^{(m)}$ \\
SAINV    & \verb=-p {sainv|6}=   & \verb=-sainv_drop [0.05]=    & The drop criterion\\
SA-AMG   & \verb=-p {saamg|7}=   & \verb=-saamg_unsym [false]=    &
 Select the unsymmetric version    \\
         &                       &                             & (The matrix structure must be symmetric)    \\
         &                       & \verb=-saamg_theta [0.05|0.12]= & The
 drop criterion $a^2_{ij}\le\theta^2|a_{ii}||a_{jj}|$ \\
         &                       &                             & (symmetric or unsymmetric) \\
Crout ILU& \verb=-p {iluc|8}=    & \verb=-iluc_drop [0.05]=    & The drop criterion    \\
         &                       & \verb=-iluc_rate [5.0]=     & The
 ratio of the maximum fill-in \\
ILUT     & \verb=-p {ilut|9}=    & \verb=-ilut_drop [0.05]=    & The drop criterion    \\
         &                       & \verb=-ilut_rate [5.0]=     & The
 ratio of the maximum fill-in \\
Additive Schwarz  & \verb=-adds true=   &  \verb=-adds_iter [1]= & The
 number of iterations   \\
\hline         
\end{tabular}
\end{center}
\end{minipage}
\\ \\
\newpage
\begin{minipage}[t]{\textwidth}
\begin{center}
{\bf Other Options}\\
\begin{tabular}{l|ll}\hline\hline
Option &                          \\ \hline
\verb=-maxiter [1000]= & The maximum number of iterations         \\ 
\verb=-tol [1.0e-12]=  & The convergence criterion              \\
\verb=-print [0]=      & The display of the residual                 \\
                       & \verb=-print {none|0}     =  None \\
                       & \verb=-print {mem|1}      =  Save the residual history\\
                       & \verb=-print {out|2}      =  Display the residual history\\
                       & \verb=-print {all|3}      =  Save the residual history and display it on the screen\\
\verb=-scale [0]=      & The scaling \\
                       & (The result will overwrite the original matrix and vectors) \\
                       & \verb=-scale {none|0}     =  No scaling \\ 
                       & \verb=-scale {jacobi|1}   =  The Jacobi scaling $D^{-1}Ax=D^{-1}b$ \\
                       & \verb=                    =  ($D$ represents the diagonal of $A=(a_{ij})$)\\
                       & \verb=-scale {symm_diag|2}=  The diagonal scaling $D^{-1/2}AD^{-1/2}x=D^{-1/2}b$ \\
                       & \verb=                    =  ($D^{-1/2}$ represents the diagonal matrix with $1/\sqrt{a_{ii}}$ \\
                       & \verb=                    =   as the diagonal) \\ 
\verb=-initx_zeros [true]= & The behavior of the initial vector $x_{0}$  \\
                       & \verb=-initx_zeros {false|0}     =  Given values \\
                       & \verb=-initx_zeros {true|1}      =  All values are set to $0$ \\
\verb=-omp_num_threads [t]= & The number of threads        \\ 
                            & (\verb=t= represents the maximum number of
 threads) \\
\verb=-storage [0]=    & The matrix storage format \\
\verb=-storage_block [2]=& The block size of the BSR and BSC formats\\ 
\verb=-f [0]=          & The precision of the linear solver\\
                       & \verb=-f {double|0}       =  Double precision \\ 
                       & \verb=-f {quad|1}         =  Quadruple precision \\
\hline         
\end{tabular}
\end{center}
\end{minipage}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\subsubsection{lis\_solver\_set\_optionC}
\begin{screen}
\verb|C       LIS_INT lis_solver_set_optionC(LIS_SOLVER solver)|\\
\verb|Fortran subroutine lis_solver_set_optionC(LIS_SOLVER solver, LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Set the options for the solver on the command line.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[None]
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt solver] The solver
\item[\tt ierr] The return code
\end{namelist}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{lis\_solve}
\begin{screen}
\verb|C       LIS_INT lis_solve(LIS_MATRIX A, LIS_VECTOR b, LIS_VECTOR x,|\\
\verb|         LIS_SOLVER solver)|\\
\verb|Fortran subroutine lis_solve(LIS_MATRIX A, LIS_VECTOR b, LIS_VECTOR x,|\\
\verb|         LIS_SOLVER solver, LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Solve the linear equation $Ax = b$ with the specified solver.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt A] The coefficient matrix
\item[\tt b] The right-hand side vector
\item[\tt x] The initial vector
\item[\tt solver] The solver
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt x] The solution
\item[\tt solver] The number of iterations, the execution time, etc.
\item[\tt ierr] The return code
\end{namelist}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\subsubsection{lis\_solve\_kernel}
\begin{screen}
\verb|C       LIS_INT lis_solve_kernel(LIS_MATRIX A, LIS_VECTOR b, LIS_VECTOR x,|\\
\verb|         LIS_SOLVER solver, LIS_PRECON, precon)|\\
\verb|Fortran subroutine lis_solve_kernel(LIS_MATRIX A, LIS_VECTOR b, LIS_VECTOR x,|\\
\verb|         LIS_SOLVER solver, LIS_PRECON precon, LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Solve the linear equation $Ax = b$ with the specified solver and the predefined preconditioner.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt A] The coefficient matrix
\item[\tt b] The right-hand side vector
\item[\tt x] The initial vector
\item[\tt solver] The solver
\item[\tt precon] The preconditioner
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt x] The solution
\item[\tt solver] The number of iterations, the execution time, etc.
\item[\tt ierr] The return code
\end{namelist}
{\bf Note}\\
\indent
See {\tt lis-(\$VERSION)/src/esolver/lis\_esolver\_ii.c}, 
which computes the smallest eigenvalue by calling 
\verb|lis_solve_kernel| multiple times, for example. 
\\
\\

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\subsubsection{lis\_solver\_get\_status}
\begin{screen}
\verb|C       LIS_INT lis_solver_get_status(LIS_SOLVER solver, LIS_INT *status)|\\
\verb|Fortran subroutine lis_solver_get_status(LIS_SOLVER solver, LIS_INTEGER status,|\\
\verb|         LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Get the status from the solver.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt solver] The solver
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt status] The status
\item[\tt ierr] The return code
\end{namelist}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{lis\_solver\_get\_iters}
\begin{screen}
\verb|C       LIS_INT lis_solver_get_iters(LIS_SOLVER solver, LIS_INT *iters)|\\
\verb|Fortran subroutine lis_solver_get_iters(LIS_SOLVER solver, LIS_INTEGER iters,|\\
\verb|         LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Get the number of iterations from the solver.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt solver] The solver
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt iters] The number of iterations
\item[\tt ierr] The return code
\end{namelist}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\subsubsection{lis\_solver\_get\_itersex}
\begin{screen}
\verb|C       LIS_INT lis_solver_get_itersex(LIS_SOLVER solver, LIS_INT *iters,|\\
\verb|         LIS_INT *iters_double, LIS_INT *iters_quad)|\\
\verb|Fortran subroutine lis_solver_get_itersex(LIS_SOLVER solver, LIS_INTEGER iters,|\\
\verb|         LIS_INTEGER iters_double, LIS_INTEGER iters_quad, LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Get the detailed information on the number of iterations from the solver.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt solver] The solver
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt iters] The number of iterations
\item[\tt iters\_double] The number of double precision iterations
\item[\tt iters\_quad] The number of quadruple precision iterations
\item[\tt ierr] The return code
\end{namelist}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{lis\_solver\_get\_time}
\begin{screen}
\verb|C       LIS_INT lis_solver_get_time(LIS_SOLVER solver, double *times)|\\
\verb|Fortran subroutine lis_solver_get_time(LIS_SOLVER solver, real*8 times,|\\
\verb|         LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Get the execution time from the solver.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt solver] The solver
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt times] The time in seconds of the execution
\item[\tt ierr] The return code
\end{namelist}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\subsubsection{lis\_solver\_get\_timeex}
\begin{screen}
\verb|C       LIS_INT lis_solver_get_timeex(LIS_SOLVER solver, double *times,|\\
\verb|         double *itimes, double *ptimes, double *p_c_times, double *p_i_times)|\\
\verb|Fortran subroutine lis_solver_get_timeex(LIS_SOLVER solver, real*8 times,|\\
\verb|         real*8 itimes, real*8 ptimes, real*8 p_c_times, real*8 p_i_times,|\\
\verb|         LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Get the detailed information on the execution time from the solver.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt solver] The solver
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt times] The total time in seconds
\item[\tt itimes] The time in seconds of the iterations
\item[\tt ptimes] The time in seconds of the preconditioning
\item[\tt p\_c\_times] The time in seconds of the creation of the preconditioner
\item[\tt p\_i\_times] The time in seconds of the iterations in the preconditioner
\item[\tt ierr] The return code
\end{namelist}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{lis\_solver\_get\_residualnorm}
\begin{screen}
\verb|C       LIS_INT lis_solver_get_residualnorm(LIS_SOLVER solver, LIS_REAL *residual)|\\
\verb|Fortran subroutine lis_solver_get_residualnorm(LIS_SOLVER solver,|\\
\verb|         LIS_REAL residual, LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Calculate the relative residual norm $||b - Ax||_2 / ||b||_2$ from the solution $x$.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt solver] The solver
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt residual]  The relative residual norm $||b - Ax||_2 / ||b||_2$
\item[\tt ierr] The return code
\end{namelist}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\subsubsection{lis\_solver\_get\_rhistory}
\begin{screen}
\verb|C       LIS_INT lis_solver_get_rhistory(LIS_SOLVER solver, LIS_VECTOR v)|\\
\verb|Fortran subroutine lis_solver_get_rhistory(LIS_SOLVER solver, LIS_VECTOR v,|\\
\verb|         LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Store the residual norm history of the solver.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt solver] The solver
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt v] The vector
\item[\tt ierr] The return code
\end{namelist}
{\bf Note}\\
\indent
Vector $v$ must be created in advance with the function \verb|lis_vector_create|. 
When vector $v$ is shorter than the residual history, 
it stores the residual history in order to vector $v$.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\subsubsection{lis\_solver\_get\_solver}
\begin{screen}
\verb|C       LIS_INT lis_solver_get_solver(LIS_SOLVER solver, LIS_INT *nsol)|\\
\verb|Fortran subroutine lis_solver_get_solver(LIS_SOLVER solver, LIS_INTEGER nsol,|\\
\verb|         LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Get the solver number from the solver.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt solver] The solver
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt nsol] The solver number
\item[\tt ierr] The return code
\end{namelist}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{lis\_solver\_get\_precon}
\begin{screen}
\verb|C       LIS_INT lis_solver_get_precon(LIS_SOLVER solver, LIS_INT *precon_type)|\\
\verb|Fortran subroutine lis_solver_get_precon(LIS_SOLVER solver, LIS_INTEGER precon_type,|\\
\verb|         LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Get the preconditioner number from the solver.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt solver] The solver
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt precon\_type] The preconditioner number
\item[\tt ierr] The return code
\end{namelist}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\subsubsection{lis\_solver\_get\_solvername}
\begin{screen}
\verb|C       LIS_INT lis_solver_get_solvername(LIS_INT nsol, char *name)|\\
\verb|Fortran subroutine lis_solver_get_solvername(LIS_INTEGER nsol, character name,|\\
\verb|         LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Get the solver name from the solver number.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt nsol] The solver number
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt name] The solver name
\item[\tt ierr] The return code
\end{namelist}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{lis\_solver\_get\_preconname}
\begin{screen}
\verb|C       LIS_INT lis_solver_get_preconname(LIS_INT precon_type, char *name)|\\
\verb|Fortran subroutine lis_solver_get_preconname(LIS_INTEGER precon_type,|\\
\verb|         character name, LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Get the preconditioner name from the preconditioner number.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt precon\_type] The preconditioner number
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt name] The preconditioner name
\item[\tt ierr] The return code
\end{namelist}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\subsection{Solving Eigenvalue Problems}
\subsubsection{lis\_esolver\_create}
\begin{screen}
\verb|C       LIS_INT lis_esolver_create(LIS_ESOLVER *esolver)|\\
\verb|Fortran subroutine lis_esolver_create(LIS_ESOLVER esolver, LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Create the eigensolver.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[None]
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt esolver] The eigensolver
\item[\tt ierr] The return code
\end{namelist}
{\bf Note}\\
\indent
\verb|esolver| has the information on the eigensolver, the preconditioner, etc.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{lis\_esolver\_destroy}
\begin{screen}
\verb|C       LIS_INT lis_esolver_destroy(LIS_ESOLVER esolver)|\\
\verb|Fortran subroutine lis_esolver_destroy(LIS_ESOLVER esolver, LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Destroy the eigensolver.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt esolver] The eigensolver to be destoyed
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt ierr] The return code
\end{namelist}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\subsubsection{lis\_esolver\_set\_option}
  \label{sec:setoptions}
\begin{screen}
\verb|C       LIS_INT lis_esolver_set_option(char *text, LIS_ESOLVER esolver)|
\verb|Fortran subroutine lis_esolver_set_option(character text, LIS_ESOLVER esolver,|\\
\verb|         LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Set the options for the eigensolver.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt text] The command line options
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt esolver] The eigensolver
\item[\tt ierr] The return code
\end{namelist}
{\bf Note}\\
\indent
The table below shows the available command line options, 
where \verb=-e {pi|1}= means \verb=-e pi= or \verb=-e 1= and \verb=-emaxiter [1000]= indicates 
that \verb=-emaxiter= defaults to $1,000$.
\\
\\
\begin{minipage}[t]{\textwidth}
\begin{center}
{\bf Options for Eigensolvers} (Default: \verb=-e pi=) \\
\begin{tabular}{l|lll}\hline\hline
 Eigensolver      & Option              &  Auxiliary Options  & \\ \hline
\hline
 Power                             & \verb=-e {pi|1}=        &    \\ 
 Inverse                           & \verb=-e {ii|2}=        &
 \verb=-i [cg]= & The linear solver \\
 Approximate Inverse               & \verb=-e {aii|3}=       & 
 \verb=-i [cg]= & The linear solver \\
 Rayleigh Quotient                 & \verb=-e {rqi|4}=       &
 \verb=-i [cg]= & The linear solver \\
 Subspace                          & \verb=-e {si|5}=        &
 \verb=-ss [2]= & The size of the subspace \\
                                             &                         &
 \verb=-m [0]= & The mode number\\
 Lanczos                           & \verb=-e {li|6}=        &
 \verb=-ss [2]= & The size of the subspace \\
                                             &                         &
 \verb=-m [0]= & The mode number\\
 CG                          & \verb=-e {cg|7}=        &    \\
 CR                          & \verb=-e {cr|8}=        &    \\
\hline         
\end{tabular}
\end{center}
\end{minipage}
\\ \\
\begin{minipage}[t]{\textwidth}
\begin{center}
{\bf Options for Preconditioners} (Default: \verb=-p none=)\\
\begin{tabular}{l|lll}\hline\hline
Preconditioner   & Option           & Auxiliary Options \\ \hline
None     & \verb=-p {none|0}=    &   \\
Jacobi   & \verb=-p {jacobi|1}=  &     \\
ILU(k)   & \verb=-p {ilu|2}=     & \verb=-ilu_fill [0]=    & The fill level $k$ \\
SSOR     & \verb=-p {ssor|3}=    & \verb=-ssor_w [1.0]=    & The relaxation coefficient $\omega$ ($0<\omega<2$) \\
Hybrid   & \verb=-p {hybrid|4}=  & \verb=-hybrid_i [sor]=  & The linear solver\\
         &                       & \verb=-hybrid_maxiter [25]= & The
 maximum number of iterations \\
         &                       & \verb=-hybrid_tol [1.0e-3]= & The convergence criterion \\
         &                       & \verb=-hybrid_w [1.5]=      & The
 relaxation coefficient $\omega$ of the SOR \\
         &                       &                             & ($0<\omega<2$) \\
         &                       & \verb=-hybrid_ell [2]=      & The
 degree $l$ of the BiCGSTAB(l) \\
         &                       & \verb=-hybrid_restart [40]= & The
 restart values of the GMRES \\
         &                       &                             & and
 Orthomin \\
I+S      & \verb=-p {is|5}=      & \verb=-is_alpha [1.0]=  &  The
 parameter $\alpha$ of $I+\alpha S^{(m)}$ \\
         &                       & \verb=-is_m [3]=        & The
 parameter $m$ of $I+\alpha S^{(m)}$ \\
SAINV    & \verb=-p {sainv|6}=   & \verb=-sainv_drop [0.05]=    & The drop criterion\\
SA-AMG   & \verb=-p {saamg|7}=   & \verb=-saamg_unsym [false]=    &
 Select the unsymmetric version    \\
         &                       &                             & (The matrix structure must be symmetric)    \\
         &                       & \verb=-saamg_theta [0.05|0.12]= & The
 drop criterion $a^2_{ij}\le\theta^2|a_{ii}||a_{jj}|$ \\
         &                       &                             & (symmetric or unsymmetric) \\
Crout ILU& \verb=-p {iluc|8}=    & \verb=-iluc_drop [0.05]=    & The drop criterion    \\
         &                       & \verb=-iluc_rate [5.0]=     & The
 ratio of the maximum fill-in \\
ILUT     & \verb=-p {ilut|9}=    & \verb=-ilut_drop [0.05]=    & The drop criterion    \\
         &                       & \verb=-ilut_rate [5.0]=     & The
 ratio of the maximum fill-in \\
Additive Schwarz  & \verb=-adds true=   &  \verb=-adds_iter [1]= & The
 number of iterations   \\
\hline         
\end{tabular}
\end{center}
\end{minipage}
\\ \\
\begin{minipage}[t]{\textwidth}
\begin{center}
{\bf Other Options}\\
\begin{tabular}{l|ll}\hline\hline
Option &                          \\ \hline
\verb=-emaxiter [1000]= & The maximum number of iterations         \\ 
\verb=-etol [1.0e-12]=  & The convergence criterion              \\
\verb=-eprint [0]=      & The display of the residual                 \\
                       & \verb=-eprint {none|0}     =  None \\
                       & \verb=-eprint {mem|1}      =  Save the residual history\\
                       & \verb=-eprint {out|2}      =  Display the residual history\\
                       & \verb=-eprint {all|3}      =  Save the residual history and display it on the screen\\
\verb=-ie [ii]= & The inner eigensolver used in the Lanczos and Subspace\\
                       & \verb=-ie {pi|1}           =  The Power (the Subspace only) \\
                       & \verb=-ie {ii|2}           =  The Inverse \\
                       & \verb=-ie {aii|3}          =  The Approximate Inverse \\
                       & \verb=-ie {rqi|4}          =  The Rayleigh Quotient \\
\verb=-shift [0.0]= & The amount of the shift  \\
\verb=-initx_ones [true]= & The behavior of the initial vector $x_{0}$  \\
                       & \verb=-initx_ones {false|0}     =  Given values \\
                       & \verb=-initx_ones {true|1}      =  All values are set to $1$ \\
\verb=-omp_num_threads [t]= & The number of threads        \\ 
                            & (\verb=t= represents the maximum number of
 threads) \\
\verb=-estorage [0]=   & The matrix storage format \\
\verb=-estorage_block [2]=& The block size of the BSR and BSC formats\\ 
\verb=-ef [0]=         & The precision of the eigensolver\\
                       & \verb=-ef {double|0}       =  Double precision \\ 
                       & \verb=-ef {quad|1}         =  Quadruple precision \\
\hline         
\end{tabular}
\end{center}
\end{minipage}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\subsubsection{lis\_esolver\_set\_optionC}
\begin{screen}
\verb|C       LIS_INT lis_esolver_set_optionC(LIS_ESOLVER esolver)|\\
\verb|Fortran subroutine lis_esolver_set_optionC(LIS_ESOLVER esolver, LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Set the options for the eigensolver on the command line.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[None]
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt esolver] The eigensolver
\item[\tt ierr] The return code
\end{namelist}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{lis\_esolve}
\begin{screen}
\verb|C       LIS_INT lis_esolve(LIS_MATRIX A, LIS_VECTOR x,|\\ 
\verb|         LIS_REAL evalue, LIS_ESOLVER esolver)|\\
\verb|Fortran subroutine lis_esolve(LIS_MATRIX A, LIS_VECTOR x,|\\
\verb|         LIS_REAL evalue, LIS_ESOLVER esolver, LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Solve the eigenvalue problem $Ax = \lambda x$ with the specified eigensolver.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt A] The matrix
\item[\tt x] The initial vector
\item[\tt esolver] The eigensolver
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt evalue] The eigenvalue of the mode specified by option \verb|-m [0]| 
\item[\tt x] The associated eigenvector
\item[\tt esolver] The number of iterations, the execution time, etc.
\item[\tt ierr] The return code
\end{namelist}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\subsubsection{lis\_esolver\_get\_status}
\begin{screen}
\verb|C       LIS_INT lis_esolver_get_status(LIS_ESOLVER esolver, LIS_INT *status)|\\
\verb|Fortran subroutine lis_esolver_get_status(LIS_ESOLVER esolver, LIS_INTEGER status,|\\
\verb|         LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Get the status from the eigensolver.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt esolver] The eigensolver
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt status] The status
\item[\tt ierr] The return code
\end{namelist}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{lis\_esolver\_get\_iters}
\begin{screen}
\verb|C       LIS_INT lis_esolver_get_iters(LIS_ESOLVER esolver, LIS_INT *iters)|\\
\verb|Fortran subroutine lis_esolver_get_iters(LIS_ESOLVER esolver, LIS_INTEGER iters,|\\
\verb|         LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Get the number of iterations from the eigensolver.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt esolver] The eigensolver
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt iters] The number of iterations
\item[\tt ierr] The return code
\end{namelist}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\subsubsection{lis\_esolver\_get\_itersex}
\begin{screen}
\verb|C       LIS_INT lis_esolver_get_itersex(LIS_ESOLVER esolver, LIS_INT *iters,|\\
\verb|         LIS_INT *iters_double, LIS_INT *iters_quad)|\\
\verb|Fortran subroutine lis_esolver_get_itersex(LIS_ESOLVER esolver, LIS_INTEGER iters,|\\
\verb|         LIS_INTEGER iters_double, LIS_INTEGER iters_quad, LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Get the detailed information on the number of iterations from the eigensolver.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt esolver] The eigensolver
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt iters] The number of iterations
\item[\tt iters\_double] The number of double precision iterations
\item[\tt iters\_quad] The number of quadruple precision iterations
\item[\tt ierr] The return code
\end{namelist}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{lis\_esolver\_get\_time}
\begin{screen}
\verb|C       LIS_INT lis_esolver_get_time(LIS_ESOLVER esolver, double *times)|\\
\verb|Fortran subroutine lis_esolver_get_time(LIS_ESOLVER esolver, real*8 times,|\\
\verb|         LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Get the execution time from the eigensolver.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt esolver] The eigensolver
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt times] The time in seconds of the execution
\item[\tt ierr] The return code
\end{namelist}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\subsubsection{lis\_esolver\_get\_timeex}
\begin{screen}
\verb|C       LIS_INT lis_esolver_get_timeex(LIS_ESOLVER esolver, double *times,|\\
\verb|         double *itimes, double *ptimes, double *p_c_times, double *p_i_times)|\\
\verb|Fortran subroutine lis_esolver_get_timeex(LIS_ESOLVER esolver, real*8 times,|\\
\verb|         real*8 itimes, real*8 ptimes, real*8 p_c_times, real*8 p_i_times,|\\
\verb|         LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Get the detailed information on the execution time from the eigensolver.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt esolver] The eigensolver
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt times] The total time in seconds
\item[\tt itimes] The time in seconds of the iterations
\item[\tt ptimes] The time in seconds of the preconditioning
\item[\tt p\_c\_times] The time in seconds of the creation of the preconditioner
\item[\tt p\_i\_times] The time in seconds of the iterations in the preconditioner
\item[\tt ierr] The return code
\end{namelist}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{lis\_esolver\_get\_residualnorm}
\begin{screen}
\verb|C       LIS_INT lis_esolver_get_residualnorm(LIS_ESOLVER esolver,|\\
\verb|         LIS_REAL *residual)|\\
\verb|Fortran subroutine lis_esolver_get_residualnorm(LIS_ESOLVER esolver,|\\
\verb|         LIS_REAL residual, LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Calculate the relative residual norm $||\lambda x-Ax||_2/\lambda$ from
the eigenvector $x$.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt esolver] The eigensolver
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt residual]  The relative residual norm $||\lambda x-Ax||_2/\lambda$
\item[\tt ierr] The return code
\end{namelist}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\subsubsection{lis\_esolver\_get\_rhistory}
\begin{screen}
\verb|C       LIS_INT lis_esolver_get_rhistory(LIS_ESOLVER esolver, LIS_VECTOR v)|\\
\verb|Fortran subroutine lis_esolver_get_rhistory(LIS_ESOLVER esolver, LIS_VECTOR v,|\\
\verb|         LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Store the residual norm history of the eigensolver.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt esolver] The eigensolver
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt v] The vector
\item[\tt ierr] The return code
\end{namelist}
{\bf Note}\\
\indent
Vector $v$ must be created in advance with the function \verb|lis_vector_create|. 
When vector $v$ is shorter than the residual history, 
it stores the residual history in order to vector $v$.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\subsubsection{lis\_esolver\_get\_evalues}
\begin{screen}
\verb|C       LIS_INT lis_esolver_get_evalues(LIS_ESOLVER esolver, LIS_VECTOR v)|\\
\verb|Fortran subroutine lis_esolver_get_evalues(LIS_ESOLVER esolver,|\\
\verb|         LIS_VECTOR v, LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Store the eigenvalues in vector $v$.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt esolver] The eigensolver
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt v]  The vector which stores the eigenvalues
\item[\tt ierr] The return code
\end{namelist}
{\bf Note}\\
\indent
Vector $v$ must be created in advance with the function \verb|lis_vector_create|. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{lis\_esolver\_get\_evectors}
\begin{screen}
\verb|C       LIS_INT lis_esolver_get_evectors(LIS_ESOLVER esolver, LIS_MATRIX A)|\\
\verb|Fortran subroutine lis_esolver_get_evectors(LIS_ESOLVER esolver,|\\
\verb|         LIS_MATRIX A, LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Store the eigenvectors in matrix $A$.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt esolver] The eigensolver
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt A]  The matrix in the CSR format which stores the eigenvectors
\item[\tt ierr] The return code
\end{namelist}
{\bf Note}\\
\indent
Matrix $A$ must be created in advance with the function \verb|lis_matrix_create|. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\subsubsection{lis\_esolver\_get\_esolver}
\begin{screen}
\verb|C       LIS_INT lis_esolver_get_esolver(LIS_ESOLVER esolver, LIS_INT *nesol)|\\
\verb|Fortran subroutine lis_esolver_get_esolver(LIS_ESOLVER esolver,|\\
\verb|         LIS_INTEGER nesol, LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Get the eigensolver number from the eigensolver.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt esolver] The eigensolver
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt nesol] The eigensolver number
\item[\tt ierr] The return code
\end{namelist}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{lis\_esolver\_get\_esolvername}
\begin{screen}
\verb|C       LIS_INT lis_esolver_get_esolvername(LIS_INT nesol, char *ename)|\\
\verb|Fortran subroutine lis_esolver_get_esolvername(LIS_INTEGER nesol, character ename,|\\
\verb|         LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Get the eigensolver name from the eigensolver number.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt nesol] The eigensolver number
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt name] The eigensolver name
\item[\tt ierr] The return code
\end{namelist}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\subsection{Operating External Files}
\subsubsection{lis\_input}
\begin{screen}
\verb|C       LIS_INT lis_input(LIS_MATRIX A, LIS_VECTOR b, LIS_VECTOR x, char *filename)|
\verb|Fortran subroutine lis_input(LIS_MATRIX A, LIS_VECTOR b, LIS_VECTOR x,|\\
\verb|         character filename, LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Read the matrix and vector data from the external file. 
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt filename] The source file
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt A] The matrix in the specified storage format
\item[\tt b] The right-hand side vector
\item[\tt x] The solution
\item[\tt ierr] The return code
\end{namelist}
{\bf Note}\\
\indent
The following file formats are supported:
\begin{itemize}
\item The extended Matrix Market format (extended to allow vector data)
\item The Harwell-Boeing format
\end{itemize}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{lis\_input\_vector}
\begin{screen}
\verb|C       LIS_INT lis_input_vector(LIS_VECTOR v, char *filename)|\\
\verb|Fortran subroutine lis_input_vector(LIS_VECTOR v, character filename,|\\
\verb|         LIS_INTEGER ierr) |
\end{screen}
{\bf Description}\\
\indent
Read the vector data from the external file.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt filename] The source file
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt v] The vector
\item[\tt ierr] The return code
\end{namelist}
{\bf Note}\\
\indent
The following file formats are supported:
\begin{itemize}
\item The PLAIN format
\item The extended Matrix Market format (extended to allow vector data)
\end{itemize}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{lis\_input\_matrix}
\begin{screen}
\verb|C       LIS_INT lis_input_matrix(LIS_MATRIX A, char *filename)|\\
\verb|Fortran subroutine lis_input_matrix(LIS_MATRIX A, character filename,|\\
\verb|         LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Read the matrix data from the external file.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt filename] The source file
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt A] The matrix in the specified storage format
\item[\tt x] The solution
\item[\tt ierr] The return code
\end{namelist}
{\bf Note}\\
\indent
The following file formats are supported:
\begin{itemize}
\item The Matrix Market format
\item The Harwell-Boeing format
\end{itemize}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{lis\_output}
\begin{screen}
\verb|C       LIS_INT lis_output(LIS_MATRIX A, LIS_VECTOR b, LIS_VECTOR x,|\\
\verb|         LIS_INT format, char *filename)|\\
\verb|Fortran subroutine lis_output(LIS_MATRIX A, LIS_VECTOR b, LIS_VECTOR x,|\\
\verb|         LIS_INTEGER format, character filename, LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Write the matrix and vector data into the external file.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt A] The matrix
\item[\tt b] The right-hand side vector (If no vector is written to the external file, then {\tt NULL} must be input.)
\item[\tt x] The solution (If no vector is written to the external file, then {\tt NULL} must be input.)
\item[\tt format] The file format
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt LIS\_FMT\_MM] The Matrix Market format
\end{namelist}
\item[\tt filename] The destination file
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt ierr] The return code
\end{namelist}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{lis\_output\_vector}
\begin{screen}
\verb|C       LIS_INT lis_output_vector(LIS_VECTOR v, LIS_INT format, char *filename)|
\verb|Fortran subroutine lis_output_vector(LIS_VECTOR v, LIS_INTEGER format,|\\
\verb|         character filename, LIS_INTEGER ierr) |
\end{screen}
{\bf Description}\\
\indent
Write the vector data into the external file.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt v] The vector
\item[\tt format] The file format
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt LIS\_FMT\_PLAIN] The PLAIN format
\item[\tt LIS\_FMT\_MM] The Matrix Market format
\end{namelist}
\item[\tt filename] The destination file
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt ierr] The return code
\end{namelist}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{lis\_output\_matrix}
\begin{screen}
\verb|C       LIS_INT lis_output_matrix(LIS_MATRIX A, LIS_INT format, char *filename)|\\
\verb|Fortran subroutine lis_output_matrix(LIS_MATRIX A, LIS_INTEGER format,|\\
\verb|         character filename, LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Write the matrix data into the external file.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt A] The matrix
\item[\tt format] The file format
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt LIS\_FMT\_MM] The Matrix Market format
\end{namelist}
\item[\tt filename] The destination file
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt ierr] The return code
\end{namelist}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\subsection{Other Functions}
\subsubsection{lis\_initialize}
\begin{screen}
\verb|C       LIS_INT lis_initialize(LIS_INT* argc, char** argv[])|\\
\verb|Fortran subroutine lis_initialize(LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Initialize the execution environment.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt argc] The number of command line arguments
\item[\tt argv] The command line argument
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt ierr] The return code
\end{namelist}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{lis\_finalize}
\begin{screen}
\verb|C       LIS_INT lis_finalize()|\\
\verb|Fortran subroutine lis_finalize(LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Finalize the execution environment.
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[None] 
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt ierr] The return code
\end{namelist}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\subsubsection{lis\_wtime}
\begin{screen}
\verb|C       double lis_wtime()|\\
\verb|Fortran function lis_wtime()|
\end{screen}
{\bf Description}\\
\indent
Measure the elapsed time. 
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[None]
\end{namelist}
{\bf Output}\\
\indent
The elapsed time in seconds from the given point is returned as the
double precision number.
\\ \\
\noindent
{\bf Note}\\
\indent
To measure the processing time, call \verb|lis_wtime| to get
the starting time, call it again to get the ending time, 
and calculate the difference. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsubsection{CHKERR}
\begin{screen}
\verb|C       void CHKERR(LIS_INT ierr)|\\
\verb|Fortran subroutine CHKERR(LIS_INTEGER ierr)|
\end{screen}
{\bf Description}\\
\indent
Check the value of the return code. 
\\ \\
\noindent
{\bf Input}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[\tt ierr] The return code
\end{namelist}
{\bf Output}
\begin{namelist}{XXXXXXXXXXXXXXXXXXXX}
\item[None] 
\end{namelist}
\noindent
{\bf Note}\\
\indent
If the value of the return code is not 0, it calls \verb|lis_finalize| 
and terminates the program. 

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\addcontentsline{toc}{section}{References}
\begin{thebibliography}{10}
\bibitem{nishida}
A. Nishida.
\newblock Experience in Developing an Open Source Scalable Software
	Infrastructure in Japan.
\newblock Lecture Notes in Computer Science 6017, pp.\ 87-98, Springer, 2010.

\bibitem{hestenes}
M. R. Hestenes and E. Stiefel.
\newblock Methods of Conjugate Gradients for Solving Linear Systems.
\newblock Journal of Research of the National Bureau of Standards, Vol.~49, No.~6, pp.\ 409--436, 1952.

\bibitem{lanczos1}
C. Lanczos.
\newblock Solution of Linear Equations by Minimized Iterations.
\newblock Journal of Research of the National Bureau of Standards, Vol.~49, No.~1, pp.\ 33--53, 1952.

\bibitem{fletcher}
R. Fletcher.
\newblock Conjugate Gradient Methods for Indefinite Systems.
\newblock Lecture Notes in Mathematics 506, pp.\ 73--89, Springer, 1976.

\bibitem{sogabe09}
T. Sogabe, M. Sugihara, and S. Zhang.
\newblock An Extension of the Conjugate Residual Method to Nonsymmetric Linear Systems.
\newblock Journal of Computational and Applied Mathematics, Vol.~226,
	No.~1, pp.\ 103--113, 2009.

\bibitem{sonneveld}
P. Sonneveld.
\newblock CGS, A Fast Lanczos-Type Solver for Nonsymmetric Linear Systems.
\newblock SIAM Journal on Scientific and Statistical Computing, Vol.~10, No.~1, pp.\ 36--52, 1989.

\bibitem{abe02}
K. Abe, T. Sogabe, S. Fujino, and S. Zhang.
\newblock A Product-Type Krylov Subspace Method Based on Conjugate Residual Method for Nonsymmetric Coefficient Matrices (in Japanese).
\newblock IPSJ Transactions on Advanced Computing Systems,  Vol.~48, No.~SIG8(ACS18), pp.\ 11--21, 2007.

\bibitem{bicgstab}
H. van der Vorst.
\newblock Bi-CGSTAB: A Fast and Smoothly Converging Variant of Bi-CG for
	the Solution of Nonsymmetric Linear Systems.
\newblock SIAM Journal on Scientific and Statistical Computing, Vol.~13, No.~2, pp.\ 631--644, 1992.

\bibitem{zhang}
S. Zhang.
\newblock Generalized Product-Type Methods Preconditionings Based on
	Bi-CG for Solving Nonsymmetric Linear Systems.
\newblock SIAM Journal on Scientific Computing, Vol.~18, No.~2, pp.\ 537--551, 1997. 

\bibitem{fujino01}
S. Fujino, M. Fujiwara, and M. Yoshida.
\newblock A Proposal of Preconditioned BiCGSafe Method with Safe Convergence.
\newblock Proceedings of The 17th IMACS World Congress on Scientific
	Computation, Applied Mathematics and Simulation, CD-ROM, 2005.

\bibitem{fujino02}
S. Fujino and Y. Onoue.
\newblock Estimation of BiCRSafe Method Based on Residual of BiCR Method (in Japanese).
\newblock IPSJ SIG Technical Report, 2007-HPC-111, pp.\ 25--30, 2007.

\bibitem{bicgstabl}
G. L. G. Sleijpen, H. A. van der Vorst, and D. R. Fokkema.
\newblock BiCGstab(l) and Other Hybrid Bi-CG Methods.
\newblock Numerical Algorithms, Vol.~7, No.~1, pp.\ 75--109, 1994.

\bibitem{tfqmr}
R. W. Freund.
\newblock A Transpose-Free Quasi-Minimal Residual Algorithm for
	Non-Hermitian Linear Systems.
\newblock SIAM Journal on Scientific Computing, Vol.~14, No.~2, pp.\ 470--482, 1993.

\bibitem{jacobi}
K. R. Biermann.
\newblock Eine unver\"offentlichte Jugendarbeit C. G. J. Jacobi \"uber
	wiederholte Funktionen.
\newblock Journal f\"ur die reine und angewandte Mathematik, Vol.~207, pp.\ 996-112, 1961.

\bibitem{orthomin}
S. C. Eisenstat, H. C. Elman, and M. H. Schultz.
\newblock Variational Iterative Methods for Nonsymmetric Systems of
	Linear Equations.
\newblock SIAM Journal on Numerical Analysis, Vol.~20, No.~2, pp.\ 345--357, 1983.

\bibitem{gauss}
C. F. Gauss.
\newblock Theoria Motus Corporum Coelestium in Sectionibus Conicis Solem. 
\newblock Perthes et Besser, 1809.

\bibitem{seidel}
L. Seidel.
\newblock \"Uber ein Verfahren, die Gleichungen, auf welche die Methode
	der kleinsten Quadrate f\"uhrt, sowie line\"are Gleichungen
	\"uberhaupt, durch successive Ann\"aherung  aufzul\"osen.
\newblock Abhandlungen der Bayerischen Akademie, Vol. ~11, pp.\ 81--108,
	1873.

\bibitem{gmres}
Y. Saad and M. H. Schultz.
\newblock GMRES: A Generalized Minimal Residual Algorithm for Solving
	Nonsymmetric Linear Systems.
\newblock SIAM Journal on Scientific and Statistical Computing, Vol.~7, No.~3, pp.\ 856--869, 1986.

\bibitem{young}
D. M. Young.
\newblock Iterative Methods for Solving Partial Difference Equations of
	Elliptic Type.
\newblock Doctoral Thesis, Harvard University, 1950.

\bibitem{frankel}
S. P. Frankel.
\newblock Convergence Rates of Iterative Treatments of Partial
	Differential Equations.
\newblock Mathematical Tables and Other Aids to Computation, Vol. 4, No.~30, 
	pp.\ 65--75, 1950.

\bibitem{fgmres}
Y. Saad.
\newblock A Flexible Inner-outer Preconditioned GMRES Algorithm.
\newblock SIAM Journal on Scientific and Statistical Computing, Vol.~14, No.~2, pp.\ 461--469, 1993.

\bibitem{idrs}
P. Sonnerveld and M. B. van Gijzen.
\newblock IDR(s): a Family of Simple and Fast Algorithms for Solving Large Nonsymmetric Systems of Linear Equations.
\newblock SIAM Journal on Scientific Computing, Vol.~31, No.~2, pp.\ 1035--1062, 2008.

\bibitem{minres}
C. C. Paige and M. A. Saunders.
\newblock Solution of Sparse Indefinite Systems of Linear Equations.
\newblock SIAM Journal on Numerical Analysis, Vol.~12, No.~4, pp.\ 617--629, 1975.

\bibitem{power}
R. von Mises and H. Pollaczek-Geiringer.
\newblock Praktische Verfahren der Gleichungsaufl\"osung.
\newblock Zeitschrift f\"ur Angewandte Mathematik und Mechanik, Vol.~9,
	No.~2, pp.\ 152--164, 1929.

\bibitem{inverse}
H. Wielandt.
\newblock Beitr\"age zur mathematischen Behandlung komplexer
	Eigenwertprobleme, Teil V: Bestimmung h\"oherer Eigenwerte durch
	gebrochene Iteration.
\newblock Bericht B 44/J/37, Aerodynamische Versuchsanstalt G\"ottingen, 1944.

\bibitem{rayleigh}
J. W. S. Rayleigh.
\newblock Some General Theorems relating to Vibrations.
\newblock Proceedings of the London Mathematical Society, Vol.~4, No.~1,
	pp.\ 357--368, 1873.

\bibitem{rutishauser}
H. R. Rutishauser.
\newblock Computational Aspects of F. L. Bauser's Simultaneous Iteration
	Method.
\newblock Numerische Mathematik, Vol.~13, No.~1, pp.\ 4--13, 1969.

\bibitem{lanczos2}
C. Lanczos.
\newblock An Iteration Method for the Solution of the Eigenvalue Problem
	of Linear Differential and Integral Operators.
\newblock Journal of Research of the National Bureau of Standards, Vol.~45, No.~4, pp.\ 255--282, 1950.

\bibitem{knyazev}
A. V. Knyazev.
\newblock Toward the Optimal Preconditioned Eigensolver: Locally Optimal
	Block Preconditioned Conjugate Gradient Method.
\newblock SIAM Journal on Scientific Computing, Vol.~23, No.~2, pp.\ 517--541, 2001.

\bibitem{suetomi}
E. Suetomi and H. Sekimoto.
\newblock Conjugate Gradient Like Methods and Their Application to
	Eigenvalue Problems for Neutron Diffusion Equation.
\newblock Annals of Nuclear Energy, Vol.~18, No.~4, pp.\ 205--227, 1991.

\bibitem{axelsson}
O. Axelsson.
\newblock A Survey of Preconditioned Iterative Methods for Linear
	Systems of Equations.
\newblock BIT, Vol.~25, No.~1, pp.\ 166--187, 1985.

\bibitem{gustafsson}
I. Gustafsson.
\newblock A Class of First Order Factorization Methods.
\newblock BIT, Vol.~18, No.~2, pp.\ 142--156, 1978.

\bibitem{ilut}
Y. Saad.
\newblock ILUT: A Dual Threshold Incomplete LU Factorization.
\newblock Numerical Linear Algebra with Applications, Vol.~1, No.~4, pp.\ 387--402, 1994. 

\bibitem{ITSOL}
Y. Saad, et al.
\newblock ITSOL: ITERATIVE SOLVERS Package. \\
\newblock http://www-users.cs.umn.edu/\textasciitilde saad/software/ITSOL/.

\bibitem{iluc}
N. Li, Y. Saad, and E. Chow.
\newblock Crout Version of ILU for General Sparse Matrices.
\newblock SIAM Journal on Scientific Computing, Vol.~25, No.~2, pp.\ 716--728, 2003. 

\bibitem{kohno01}
T. Kohno, H. Kotakemori, and H. Niki.
\newblock Improving the Modified Gauss-Seidel Method for Z-matrices.
\newblock Linear Algebra and its Applications, Vol.~267, pp.\ 113--123, 1997.

\bibitem{fujii01}
A. Fujii, A. Nishida, and Y. Oyanagi.
\newblock Evaluation of Parallel Aggregate Creation Orders : Smoothed Aggregation Algebraic Multigrid Method.
\newblock High Performance Computational Science and Engineering, pp.\ 99--122, Springer, 2005. 

\bibitem{abe01}
K. Abe, S. Zhang, H. Hasegawa, and R. Himeno.
\newblock A SOR-base Variable Preconditioned CGR Method (in Japanese).
\newblock Transactions of the JSIAM,  Vol.~11, No.~4, pp.\ 157--170, 2001.

\bibitem{bridson01}
R. Bridson and W. P. Tang.
\newblock Refining an Approximate Inverse.
\newblock Journal of Computational and Applied Mathematics, Vol.~123, No.~1-2, pp.\ 293--306, 2000. 

\bibitem{chan}
T. Chan and T. Mathew.
\newblock Domain Decomposition Algorithms.
\newblock Acta Numerica, Vol.~3, pp.\ 61--143, 1994. 

\bibitem{dryja}
M. Dryja and O. B. Widlund.
\newblock Domain Decomposition Algorithms with Small Overlap.
\newblock SIAM Journal on Scientific Computing, Vol.~15, No.~3, pp.\ 604--620, 1994.

\bibitem{kota1}
H. Kotakemori, H. Hasegawa, and A. Nishida.
\newblock Performance Evaluation of a Parallel Iterative Method Library using OpenMP.
\newblock Proceedings of the 8th International Conference on High
	Performance Computing in Asia Pacific Region, pp.\ 432--436, IEEE, 2005.

\bibitem{kota2}
H. Kotakemori, H. Hasegawa, T. Kajiyama, A. Nukada, R. Suda, and A. Nishida.
\newblock Performance Evaluation of Parallel Sparse Matrix-Vector Products on SGI Altix 3700.
\newblock Lecture Notes in Computer Science 4315, pp.\ 153--163, Springer, 2008.

\bibitem{dd}
D. H. Bailey.
\newblock A Fortran-90 Double-Double Library.
\newblock http://crd-legacy.lbl.gov/\textasciitilde dhbailey/mpdist/.

\bibitem{qd}
Y. Hida, X. S. Li, and D. H. Bailey.
\newblock Algorithms for Quad-Double Precision Floating Point Arithmetic.
\newblock Proceedings of the 15th Symposium on Computer Arithmetic, pp.\ 155--162, 2001.

\bibitem{dekker}
T. Dekker.
\newblock A Floating-Point Technique for Extending the Available Precision.
\newblock Numerische Mathematik, Vol.~18, No.~3, pp.\ 224--242, 1971.

\bibitem{Knuth}
D. E. Knuth.
The Art of Computer Programming: Seminumerical Algorithms, Vol. 2.
Addison-Wesley, 1969.

\bibitem{Bailey:High-Precision}
D. H. Bailey.
\newblock High-Precision Floating-Point Arithmetic in Scientific
	Computation.
\newblock Computing in Science and Engineering, Vol.~7, No.~3, pp.\ 54--61, IEEE, 2005.

\bibitem{intel}
\newblock Intel Fortran Compiler for Linux Systems User's Guide, Vol I.
\newblock Intel Corporation, 2004.

\bibitem{quadlis}
H. Kotakemori, A. Fujii, H. Hasegawa, and A. Nishida.
\newblock Implementation of Fast Quad Precision Operation and
	Acceleration with SSE2 for Iterative Solver Library (in
	Japanese).
\newblock IPSJ Transactions on Advanced Computing Systems,  Vol.~1, No.~1, pp.\ 73--84, 2008.

\bibitem{Courant89}
R. Courant and D. Hilbert.
\newblock Methods of Mathematical Physics. 
\newblock Wiley-VCH, 1989.

\bibitem{Lanczos70}
C. Lanczos. 
\newblock The Variational Principles of Mechanics, 4th Edition. 
\newblock University of Toronto Press, 1970.

\bibitem{Wilkinson88}
J. H. Wilkinson. 
\newblock The Algebraic Eigenvalue Problem. 
\newblock Oxford University Press, 1988.

\bibitem{young71}
D. M. Young. 
\newblock Iterative Solution of Large Linear Systems. 
\newblock Academic Press, 1971.

\bibitem{golub96}
G. H. Golub and C. F. Van Loan.
\newblock Matrix Computations, 3rd Edition.
\newblock The Johns Hopkins University Press, 1996.

\bibitem{dongarra91}
J. J. Dongarra, I. S. Duff, D. C. Sorensen, and H. A. van der Vorst.
\newblock Solving Linear Systems on Vector and Shared Memory Computers.
\newblock SIAM, 1991.

\bibitem{Saad92}
Y. Saad.
\newblock Numerical Methods for Large Eigenvalue Problems. 
\newblock Halsted Press, 1992.

\bibitem{Bar00}
R. Barrett, et al.
\newblock Templates for the Solution of Linear Systems: Building Blocks for Iterative Methods.
\newblock SIAM, 1994.

\bibitem{Saad03}
Y. Saad.
\newblock Iterative Methods for Sparse Linear Systems. Second Edition. 
\newblock SIAM, 2003.

\bibitem{greenbaum}
A. Greenbaum.
\newblock Iterative Methods for Solving Linear Systems.
\newblock SIAM, 1997.

\bibitem{bai00}
Z. Bai, et al.
\newblock Templates for the Solution of Algebraic Eigenvalue Problems.
\newblock SIAM, 2000.

\bibitem{wilkinson71}
J. H. Wilkinson and C. Reinsch.
\newblock Handbook for Automatic Computation, Vol. 2: Linear Algebra.
\newblock Grundlehren Der Mathematischen Wissenschaften, Vol. 186, Springer, 1971.

\bibitem{eispack}
B. T. Smith, J. M. Boyle, Y. Ikebe, V. C. Klema, and C. B. Moler.
\newblock Matrix Eigensystem Routines: EISPACK Guide, 2nd ed.
\newblock Lecture Notes in Computer Science 6, Springer, 1970.

\bibitem{eispack2}
B. S. Garbow, J. M. Boyle, J. J. Dongarra, and C. B. Moler.
\newblock Matrix Eigensystem Routines: EISPACK Guide Extension.
\newblock Lecture Notes in Computer Science 51, Springer, 1972.

\bibitem{linpack}
J. J. Dongarra, J. R. Bunch, G. B. Moler, and G. M. Stewart.
\newblock LINPACK Users' Guide.
\newblock SIAM, 1979.

\bibitem{rice85}
J. R. Rice and R. F. Boisvert.
\newblock Solving Elliptic Problems Using ELLPACK.
\newblock Springer, 1985.

\bibitem{lapack}
E. Anderson, et al. 
\newblock LAPACK Users' Guide. 3rd ed.
\newblock SIAM, 1987.

\bibitem{dongarra92}
J. Dongarra, A. Lumsdaine, R. Pozo, and K. Remington.
\newblock A Sparse Matrix Library in C++ for High Performance Architectures.
\newblock Proceedings of the Second Object Oriented Numerics Conference,
	pp.\ 214--218, 1992.

\bibitem{duff92}
I. S. Duff, R. G. Grimes, and J. G. Lewis.
\newblock Users' Guide for the Harwell-Boeing Sparse Matrix Collection
(Release I).
\newblock Technical Report TR/PA/92/86, CERFACS, 1992.

\bibitem{Saad90}
Y. Saad.
\newblock SPARSKIT: A Basic Tool Kit for Sparse Matrix Computations,
  Version 2, 1994.
\newblock http://www-users.cs.umn.edu/\textasciitilde saad/software/SPARSKIT/.

\bibitem{pvm}
A. Geist, et al.
\newblock PVM: Parallel Virtual Machine.
\newblock MIT Press, 1994.

\bibitem{bramley95splib}
R. Bramley and X. Wang.
\newblock SPLIB: A Library of Iterative Methods for Sparse Linear System.
\newblock Technical Report, Department of Computer Science, Indiana University, 1995.

\bibitem{matrixmarket}
R. F. Boisvert, et al. 
\newblock The Matrix Market Exchange Formats: Initial Design. 
\newblock Technical Report NISTIR 5935, National Institute of Standards and
	Technology, 1996.

\bibitem{scalapack}
L. S. Blackford, et al.
\newblock ScaLAPACK Users' Guide.
\newblock SIAM, 1997.

\bibitem{ARPACK}
R. B. Lehoucq, D. C. Sorensen, and C. Yang.
\newblock ARPACK Users' Guide: Solution of Large-Scale Eigenvalue Problems with Implicitly-Restarted Arnoldi Methods. 
\newblock SIAM, 1998. 

\bibitem{Aztec}
R. S. Tuminaro, et al.
\newblock Official Aztec User's Guide, Version 2.1.
\newblock Technical Report SAND99-8801J, Sandia National Laboratories, 1999.

\bibitem{mpi}
W. Gropp, E. Lusk, and A. Skjellum.
\newblock Using MPI, 2nd Edition: Portable Parallel Programming with the
	Message-Passing Interface.
\newblock MIT Press, 1999.

\bibitem{PETSc}
S. Balay, et al.
\newblock PETSc Users Manual.
\newblock Technical Report ANL-95/11, Argonne National Laboratory, 2004.

\bibitem{SLEPc}
V. Hernandez, J. E. Roman, and V. Vidal.
\newblock SLEPc: A Scalable and Flexible Toolkit for the Solution of
	Eigenvalue Problems.
\newblock ACM Transactions on Mathematical Software,  Vol.~31, No.~3, pp.\ 351--362, 2005.

\bibitem{Trilinos}
M. A. Heroux, et al.
\newblock An Overview of the Trilinos Project.
\newblock ACM Transactions on Mathematical Software,  Vol.~31, No.~3, pp.\ 397--423, 2005.

\bibitem{Hypre}
R. D. Falgout, J. E. Jones, and U. M. Yang.
\newblock The Design and Implementation of hypre, a Library of Parallel
	High Performance Preconditioners.
\newblock Lecture Notes in Computational Science and Engineering 51, pp.\ 209--236, Springer, 2006.

\bibitem{openmp}
B. Chapman, G. Jost, and R. van der Pas.
\newblock Using OpenMP: Portable Shared Memory Parallel Programming.
\newblock MIT Press, 2007.

\end{thebibliography}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\newpage
\appendix
\section{File Formats}
\label{sec:matinp}
This section describes the file formats available for the library.
\subsection{Extended Matrix Market Format}
The Matrix Market format does not support 
the vector data. The extended Matrix Market format is the extension of
the Matrix Market format to handle 
the matrix and vector data. Assume that the number of nonzero elements of
matrix $A=(a_{ij})$ of size $M$ $\times$ $N$ is $L$ and 
that $a_{ij} = A(I,J)$. The format is as follows: 

{\small
\begin{verbatim}
%%MatrixMarket matrix coordinate real general  <-- Header
%                                              <-+ 
%                                                | Comment lines with 0 or more lines
%                                              <-+
M N L B X                                      <-- Numbers of rows, columns, and 
I1 J1 A(I1,J1)                                 <-+    nonzero elements (0 or 1) (0 or 1)
I2 J2 A(I2,J2)                                   | Row and column number values
 . . .                                           | The index is one origin
IL JL A(IL,JL)                                 <-+
I1 B(I1)                                       <-+
I2 B(I2)                                         | Exists only when B=1
 . . .                                           | Row number value
IM B(IM)                                       <-+
I1 X(I1)                                       <-+
I2 X(I2)                                         | Exists only when X=1
 . . .                                           | Row number value
IM X(IM)                                       <-+
\end{verbatim}
}

The extended Matrix Market format for matrix $A$ and vector $b$ in Equation (\ref{eq:matavecv}) is as follows: 
\begin{equation}
A = 
\left(
\begin{array}{cccc}
2 & 1 &   &    \\
1 & 2 & 1 &    \\ 
  & 1 & 2 & 1 \\
  &   & 1 & 2 
\end{array}
\right)
~~~~~~~~~~~~
b = 
\left(
\begin{array}{c}
0 \\
1 \\ 
2 \\
3  
\end{array}
\right)
\label{eq:matavecv}
\end{equation}

{\small
\begin{verbatim}
%%MatrixMarket matrix coordinate real general
4 4 10 1 0
1 2  1.00e+00
1 1  2.00e+00
2 3  1.00e+00
2 1  1.00e+00
2 2  2.00e+00
3 4  1.00e+00
3 2  1.00e+00
3 3  2.00e+00
4 4  2.00e+00
4 3  1.00e+00
1  0.00e+00
2  1.00e+00
3  2.00e+00
4  3.00e+00
\end{verbatim}
}

\subsection{Harwell-Boeing Format}
The Harwell-Boeing format stores the matrix in the CSC format.
Assume that the array {\tt value} stores the values of the nonzero elements of matrix $A$, 
the array {\tt index} stores the row indices of the nonzero elements and 
the array {\tt ptr} stores pointers to the top of each column in the
arrays {\tt value} and {\tt index}. The format is as follows: 

{\small
\begin{verbatim}
Line 1 (A72,A8)
   1 - 72 Title
  73 - 80 Key
Line 2 (5I14)
   1 - 14 Total number of lines excluding header 
  15 - 28 Number of lines for ptr
  29 - 42 Number of lines for index
  43 - 56 Number of lines for value
  57 - 70 Number of lines for right-hand side vectors
Line 3 (A3,11X,4I14)
   1 -  3 Matrix type 
            Col.1: R Real matrix
                   C Complex matrix (Not supported)
                   P Pattern only (Not supported)
            Col.2: S Symmetric
                   U Unsymmetric
                   H Hermitian (Not supported)
                   Z Skew symmetric (Not supported)
                   R Rectangular (Not supported)
            Col.3: A Assembled
                   E Elemental matrices (Not supported)
   4 - 14 Blank space
  15 - 28 Number of rows
  29 - 42 Number of columns 
  43 - 56 Number of nonzero elements
  57 - 70 0
Line 4 (2A16,2A20)
   1 - 16 Format for ptr
  17 - 32 Format for index
  33 - 52 Format for value
  53 - 72 Format for right-hand side vectors
Line 5 (A3,11X,2I14) Only presents if there are right-hand side vectors
   1      right-hand side vector type
            F for full storage
            M for same format as matrix (Not supported)
   2      G if a starting vector is supplied
   3      X if an exact solution is supplied
   4 - 14 Blank space
  15 - 28 Number of right-hand side vectors 
  29 - 42 Number of nonzero elements
\end{verbatim}
}

The Harwell-Boeing format for matrix $A$ and vector $b$ in Equation (\ref{eq:matavecv}) is as follows: 

{\small
\begin{verbatim}
1--------10--------20--------30--------40--------50--------60--------70--------80
Harwell-Boeing format sample                                            Lis
             8             1             1             4             2
RUA                        4             4            10             4
(11i7)          (13i6)          (3e26.18)           (3e26.18)
F                          1             0
      1      3      6      9
     1     2     1     2     3     2     3     4     3     4
  2.000000000000000000E+00  1.000000000000000000E+00  1.000000000000000000E+00
  2.000000000000000000E+00  1.000000000000000000E+00  1.000000000000000000E+00
  2.000000000000000000E+00  1.000000000000000000E+00  1.000000000000000000E+00
  2.000000000000000000E+00
  0.000000000000000000E+00  1.000000000000000000E+00  2.000000000000000000E+00
  3.000000000000000000E+00
\end{verbatim}
}

\subsection{Extended Matrix Market Format for Vectors}
The extended Matrix Market format for vectors is 
the extension of the Matrix Market format to handle the vector data. 
Assume that vector $b=(b_{i})$ is a vector of size 
$N$ and that $b_{i} = B(I)$. The format is as follows:

{\small
\begin{verbatim}
%%MatrixMarket vector coordinate real general  <--  Header
%                                              <-+ 
%                                                | Comment lines with 0 or more lines
%                                              <-+
N                                              <-- Number of rows
I1 B(I1)                                       <-+
I2 B(I2)                                         | Row number value
 . . .                                           | The index is one origin
IN B(IN)                                       <-+ 
\end{verbatim}
}

The extended Matrix Market format for vector $b$ in Equation (\ref{eq:matavecv}) is as follows:

{\small
\begin{verbatim}
%%MatrixMarket vector coordinate real general
4
1  0.00e+00
2  1.00e+00
3  2.00e+00
4  3.00e+00
\end{verbatim}
}

\subsection{PLAIN Format for Vectors}
The PLAIN format for vectors is designed to write vector values in
order. Assume that vector $b=(b_{i})$ is a vector of size $N$ and
that $b_{i}$ is equal to $B(I)$. The format is as follows:

{\small
\begin{verbatim}
B(1)                                       <-+
B(2)                                         | Vector value
. . .                                        | 
B(N)                                       <-+ 
\end{verbatim}
}

The PLAIN format for vector $b$ in Equation (\ref{eq:matavecv}) is as follows: 

{\small
\begin{verbatim}
0.00e+00
1.00e+00
2.00e+00
3.00e+00
\end{verbatim}
}
\end{document}


