/* Copyright (C) The Scalable Software Infrastructure Project. All rights reserved.

   Redistribution and use in source and binary forms, with or without
   modification, are permitted provided that the following conditions are met:
   1. Redistributions of source code must retain the above copyright
      notice, this list of conditions and the following disclaimer.
   2. Redistributions in binary form must reproduce the above copyright
      notice, this list of conditions and the following disclaimer in the
      documentation and/or other materials provided with the distribution.
   3. Neither the name of the project nor the names of its contributors 
      may be used to endorse or promote products derived from this software 
      without specific prior written permission.

   THIS SOFTWARE IS PROVIDED BY THE SCALABLE SOFTWARE INFRASTRUCTURE PROJECT
   ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
   TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
   PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE SCALABLE SOFTWARE INFRASTRUCTURE
   PROJECT BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
   OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
   POSSIBILITY OF SUCH DAMAGE.
*/

/*
 * This subroutine is made based on ITSOL.
 *
 * http://www-users.cs.umn.edu/~saad/software/ITSOL/
 *
 */

#ifdef HAVE_CONFIG_H
  #include "lis_config.h"
#else
#ifdef HAVE_CONFIG_WIN_H
  #include "lis_config_win.h"
#endif
#endif

#include <stdio.h>
#include <stdlib.h>
#ifdef HAVE_MALLOC_H
        #include <malloc.h>
#endif
#include <string.h>
#include <stdarg.h>
#include <math.h>
#ifdef _OPENMP
  #include <omp.h>
#endif
#ifdef USE_MPI
  #include <mpi.h>
#endif
#include "lislib.h"

/************************************************
 * lis_precon_create
 * lis_psolve
 * lis_psolvet
 ************************************************/

#define BSR_ENABLE 0 
#undef __FUNC__
#define __FUNC__ "lis_precon_create_iluc"
LIS_INT lis_precon_create_iluc(LIS_SOLVER solver, LIS_PRECON precon)
{
  LIS_INT        storage,block,err;
  LIS_MATRIX    A,B;

  LIS_DEBUG_FUNC_IN;

#ifdef BSR_ENABLE
  storage     = solver->options[LIS_OPTIONS_STORAGE];
  block       = solver->options[LIS_OPTIONS_STORAGE_BLOCK];

  if( storage==LIS_MATRIX_BSR )
  {
    if( solver->A->matrix_type!=storage )
    {
      err = lis_matrix_convert_self(solver);
      if( err ) return err;
    }
  }
#endif

  switch( solver->A->matrix_type )
  {
  case LIS_MATRIX_CSR:
    err = lis_matrix_split(solver->A);
    if( err )
    {
      return err;
    }
    err = lis_precon_create_iluc_csr(solver,precon);
    break;
#ifdef BSR_ENABLE
  case LIS_MATRIX_BSR:
    err = lis_matrix_split(solver->A);
    if( err )
    {
      return err;
    }
    err = lis_precon_create_iluc_bsr(solver,precon);
    lis_psolve_xxx[LIS_PRECON_TYPE_ILUC]  = lis_psolve_iluc_bsr;
    break;
#endif
  default:
    A = solver->A;
    err = lis_matrix_duplicate(A,&B);
    if( err ) return err;
    lis_matrix_set_type(B,LIS_MATRIX_CSR);
    err = lis_matrix_convert(A,B);
    if( err ) return err;
    solver->A = B;
    err = lis_matrix_split(solver->A);
    if( err )
    {
      return err;
    }
    err = lis_precon_create_iluc_csr(solver,precon);
    lis_matrix_destroy(B);
    solver->A = A;
    break;
  }

  LIS_DEBUG_FUNC_OUT;
    return LIS_SUCCESS;
}


#undef __FUNC__
#define __FUNC__ "lis_precon_create_iluc_csr"
LIS_INT lis_precon_create_iluc_csr(LIS_SOLVER solver, LIS_PRECON precon)
{
#ifdef _OPENMP
  LIS_INT        err;
  LIS_INT        i,j,k,l,ii,jj,kk,ll;
  LIS_INT        is,ie,my_rank,nprocs;
  LIS_INT        n,nnz,annz,lfil,len;
  LIS_INT        cz,cw;
  LIS_INT        *iw,*iw2,*wc,*wl,*iz,*zc,*zl;
  LIS_INT        *index,*ptr;
  LIS_SCALAR    gamma,val,t,dd,tol,toldd;
  LIS_SCALAR    *z,*w,*tmp;
  LIS_SCALAR    *value;
  LIS_SCALAR    m;
  LIS_MATRIX    A;
  LIS_MATRIX_ILU  L,U;
  LIS_VECTOR    D;

  LIS_DEBUG_FUNC_IN;


  A      = solver->A;
  n      = A->n;
  tol    = solver->params[LIS_PARAMS_DROP-LIS_OPTIONS_LEN];
  m      = solver->params[LIS_PARAMS_RATE-LIS_OPTIONS_LEN];
  gamma  = solver->params[LIS_PARAMS_GAMMA-LIS_OPTIONS_LEN];
  annz   = 10+A->nnz / A->n;
  lfil   = (LIS_INT)((double)A->nnz/(2.0*n))*m;
  nprocs = omp_get_max_threads();

  L      = NULL;
  U      = NULL;


  err = lis_matrix_ilu_create(n,1,&L);
  if( err ) return err;
  err = lis_matrix_ilu_create(n,1,&U);
  if( err ) return err;
  err = lis_matrix_ilu_setCR(L);
  if( err ) return err;
  err = lis_matrix_ilu_setCR(U);
  if( err ) return err;
  err = lis_vector_duplicate(A,&D);
  if( err )
  {
    return err;
  }

  tmp   = (LIS_SCALAR *)lis_malloc(nprocs*n*sizeof(LIS_SCALAR),"lis_precon_create_iluc_csr::tmp");
  if( tmp==NULL )
  {
    LIS_SETERR_MEM(n*sizeof(LIS_SCALAR));
    return LIS_OUT_OF_MEMORY;
  }
  w   = (LIS_SCALAR *)lis_malloc(n*sizeof(LIS_SCALAR),"lis_precon_create_iluc_csr::w");
  if( w==NULL )
  {
    LIS_SETERR_MEM(n*sizeof(LIS_SCALAR));
    return LIS_OUT_OF_MEMORY;
  }
  z   = (LIS_SCALAR *)lis_malloc(n*sizeof(LIS_SCALAR),"lis_precon_create_iluc_csr::z");
  if( z==NULL )
  {
    LIS_SETERR_MEM(n*sizeof(LIS_SCALAR));
    return LIS_OUT_OF_MEMORY;
  }

  iw   = (LIS_INT *)lis_malloc(nprocs*n*sizeof(LIS_INT),"lis_precon_create_iluc_csr::iw");
  if( iw==NULL )
  {
    LIS_SETERR_MEM(nprocs*n*sizeof(LIS_INT));
    return LIS_OUT_OF_MEMORY;
  }
  iw2  = (LIS_INT *)lis_malloc( nprocs*(n+1)*sizeof(LIS_INT), "lis_precon_create_iluc_csr::iw2" );
  if( iw2==NULL )
  {
    LIS_SETERR_MEM(nprocs*(n+1)*sizeof(LIS_INT));
    return LIS_OUT_OF_MEMORY;
  }
  wc   = (LIS_INT *)lis_malloc(n*sizeof(LIS_INT),"lis_precon_create_iluc_csr::wc");
  if( wc==NULL )
  {
    LIS_SETERR_MEM(n*sizeof(LIS_INT));
    return LIS_OUT_OF_MEMORY;
  }
  wl   = (LIS_INT *)lis_malloc(n*sizeof(LIS_INT),"lis_precon_create_iluc_csr::wl");
  if( wl==NULL )
  {
    LIS_SETERR_MEM(n*sizeof(LIS_INT));
    return LIS_OUT_OF_MEMORY;
  }
  iz   = (LIS_INT *)lis_malloc(nprocs*n*sizeof(LIS_INT),"lis_precon_create_iluc_csr::iz");
  if( iw==NULL )
  {
    LIS_SETERR_MEM(n*sizeof(LIS_INT));
    return LIS_OUT_OF_MEMORY;
  }
  zc   = (LIS_INT *)lis_malloc(n*sizeof(LIS_INT),"lis_precon_create_iluc_csr::zc");
  if( wc==NULL )
  {
    LIS_SETERR_MEM(n*sizeof(LIS_INT));
    return LIS_OUT_OF_MEMORY;
  }
  zl   = (LIS_INT *)lis_malloc(n*sizeof(LIS_INT),"lis_precon_create_iluc_csr::zl");
  if( wl==NULL )
  {
    LIS_SETERR_MEM(n*sizeof(LIS_INT));
    return LIS_OUT_OF_MEMORY;
  }

  nnz = A->L->nnz;
  err = lis_matrix_malloc_csr(n,nnz,&ptr,&index,&value);


  #pragma omp parallel private(i,ii,j,jj,k,kk,l,ll,is,ie,my_rank,cz,cw,toldd,t,nnz,len)
  {
    my_rank  = omp_get_thread_num();
    LIS_GET_ISIE(my_rank,nprocs,n,is,ie);

    
    #pragma omp for
    for(i=0;i<nprocs;i++)
    {
      memset( &iw2[i*n], 0, n*sizeof(LIS_INT) );
    }
    #pragma omp for
    for(i=0;i<=n;i++)
    {
      ptr[i] = 0;
    }
    #pragma omp for
    for(i=0;i<n;i++)
    {
      for(j=A->L->ptr[i];j<A->L->ptr[i+1];j++)
      {
        jj           = A->L->index[j];
/*        if( jj>=is && jj<ie )*/
        {
          iw2[my_rank*n + jj]++;
        }
      }
    }
    #pragma omp for
    for(j=0;j<n;j++)
    {
      k = 0;
      for(i=0;i<nprocs;i++)
      {
        k += iw2[i*n+j];
      }
      iw[j] = k;
    }
    #pragma omp single
    for(j=0;j<n;j++)
    {
      ptr[j+1] = ptr[j] + iw[j];
    }
    #pragma omp for
    for(j=0;j<n;j++)
    {
      k = ptr[j];
      for(i=0;i<nprocs;i++)
      {
        l = iw2[i*n+j];
        iw2[i*n+j] = k;
        k = l + k;
      }
    }
    #pragma omp for
    for(i=0;i<n;i++)
    {
      for(j=ptr[i];j<ptr[i+1];j++)
      {
        value[j] = 0.0;
        index[j] = 0;
      }
    }
    #pragma omp for
    for(i=0;i<n;i++)
    {
      for(j=A->L->ptr[i];j<A->L->ptr[i+1];j++)
      {
        k        = A->L->index[j];
/*        if( k>=is && k<ie )*/
        {
          l        = iw2[my_rank*n+k];
          value[l] = A->L->value[j];
          index[l] = i;
          iw2[my_rank*n+k]++;
        }
      }
    }
    

    for(i=is;i<ie;i++)
    {
      D->value[i] = A->D->value[i];
      zl[i]       = -1;
      wl[i]       = -1;
      zc[i]       = 0;
      wc[i]       = 0;
    }


  for(k=is;k<ie;k++)
  {
    cz = 0;
    cw = 0;
    /* z_j = a_kj (j>=k) */
    for(j=A->U->ptr[k];j<A->U->ptr[k+1];j++)
    {
      jj       = A->U->index[j];
      if( jj<is || jj>=ie ) continue;
      #ifdef USE_MPI
        if( jj>n-1 ) continue;
      #endif
      z[jj]    = A->U->value[j];
      iz[my_rank*n+cz++] = jj;
      zc[jj]   = 1;
    }

    /* w_j = a_jk (j>k) */
    for(j=ptr[k];j<ptr[k+1];j++)
    {
      jj       = index[j];
      if( jj<is || jj>=ie ) continue;
      w[jj]    = value[j];
      iw[my_rank*n+cw++] = jj;
      wc[jj]   = 1;
    }

    /* z_j = z_j - l_ki*u_ij */
    i = wl[k];
    while( i>=is )
    {
      ii  = wc[i];
      kk  = zc[i];
      val = L->value[i][ii];
      for(j=kk;j<U->nnz[i];j++)
      {
        jj     = U->index[i][j];
        if( jj==k ) continue;
        if( zc[jj]==1 )
        {
          z[jj]   -= val*U->value[i][j];
        }
        else
        {
          z[jj]    = -val*U->value[i][j];
          iz[my_rank*n+cz++] = jj;
          zc[jj]   = 1;
        }
      }
      ii++;
      wc[i] = ii;
      if( ii<L->nnz[i] )
      {
        kk     = L->index[i][ii];
        ll     = i;
        i      = wl[ll];
        wl[ll] = wl[kk];
        wl[kk] = ll;
      }
      else
      {
        i      = wl[i];
      }
    }

    /* w_j = w_j - l_ji*u_ik */
    i = zl[k];
    while( i>=is )
    {
      ii  = zc[i];
      kk  = wc[i];
      val = U->value[i][ii];
      for(j=kk;j<L->nnz[i];j++)
      {
        jj     = L->index[i][j];
        if( wc[jj]==1 )
        {
          w[jj]   -= val*L->value[i][j];
        }
        else
        {
          w[jj]    = -val*L->value[i][j];
          iw[my_rank*n+cw++] = jj;
          wc[jj]   = 1;
        }
      }
      ii++;
      zc[i] = ii;
      if( ii<U->nnz[i] )
      {
        kk     = U->index[i][ii];
        ll     = i;
        i      = zl[ll];
        zl[ll] = zl[kk];
        zl[kk] = ll;
      }
      else
      {
        i      = zl[i];
      }
    }

    toldd       = fabs(D->value[k])*tol;
    D->value[k] = 1.0/D->value[k];
    t           = D->value[k];
    if( cz<cw )
    {
      for(j=0;j<cz;j++)
      {
        jj = iz[my_rank*n+j];
        if( wc[jj] )
        {
          D->value[jj] -= z[jj]*w[jj]*t;
        }
      }
    }
    else
    {
      for(j=0;j<cw;j++)
      {
        jj = iw[my_rank*n+j];
        if( zc[jj] )
        {
          D->value[jj] -= z[jj]*w[jj]*t;
        }
      }
    }
    /* drop U */
    nnz = 0;
    for(j=0;j<cz;j++)
    {
      jj = iz[my_rank*n+j];
      if( fabs(z[jj])>toldd )
      {
        iz[my_rank*n+nnz++] = jj;
      }
      else
      {
        z[k] = z[k] + fabs(z[jj]);
        zc[jj] = 0;
      }
    }
    len = _min(lfil,nnz);
    for(j=0;j<nnz;j++) tmp[my_rank*n+j] = fabs(z[is+j]);
    lis_sort_di(0,nnz-1,&tmp[my_rank*n],&iz[my_rank*n]);
    lis_sort_i(0,len-1,&iz[my_rank*n]);
    U->nnz[k] = len;
    if( len>0 )
    {
      U->index[k] = (LIS_INT *)malloc(len*sizeof(LIS_INT));
      U->value[k] = (LIS_SCALAR *)malloc(len*sizeof(LIS_SCALAR));
    }
    for(j=0;j<len;j++)
    {
      jj = iz[my_rank*n+j];
      U->index[k][j] = jj;
      U->value[k][j] = z[jj];
    }
    for(j=len;j<cz;j++) zc[iz[my_rank*n+j]] = 0;
    cz = nnz;

    /* drop L */
    nnz = 0;
    for(j=0;j<cw;j++)
    {
      jj = iw[my_rank*n+j];
      if( fabs(w[jj])>toldd )
      {
        iw[my_rank*n+nnz++] = jj;
      }
      else
      {
        wc[jj] = 0;
      }
    }
    len = _min(lfil,nnz);
    for(j=0;j<nnz;j++) tmp[my_rank*n+j] = fabs(w[is+j]);
    lis_sort_di(0,nnz-1,&tmp[my_rank*n],&iw[my_rank*n]);
    lis_sort_i(0,len-1,&iw[my_rank*n]);
    L->nnz[k] = len;
    if( len>0 )
    {
      L->index[k] = (LIS_INT *)malloc(len*sizeof(LIS_INT));
      L->value[k] = (LIS_SCALAR *)malloc(len*sizeof(LIS_SCALAR));
    }
    for(j=0;j<len;j++)
    {
      jj = iw[my_rank*n+j];
      L->index[k][j] = jj;
      L->value[k][j] = w[jj]*t;
    }
    for(j=len;j<cw;j++) wc[iw[my_rank*n+j]] = 0;
    cw = nnz;

    /**/
    for(j=0;j<cz;j++) zc[iz[my_rank*n+j]] = 0;
    for(j=0;j<cw;j++) wc[iw[my_rank*n+j]] = 0;
    if(U->nnz[k]>0)
    {
      jj     = iz[my_rank*n+0];
      zc[k]  = 0;
      zl[k]  = zl[jj];
      zl[jj] = k;
    }
    if(L->nnz[k]>0)
    {
      jj     = iw[my_rank*n+0];
      wc[k]  = 0;
      wl[k]  = wl[jj];
      wl[jj] = k;
    }
  }
  }

  precon->L  = L;
  precon->U  = U;
  precon->D  = D;

  lis_free2(13,tmp,w,z,iw,iw2,wc,wl,iz,zc,zl,ptr,index,value);

  LIS_DEBUG_FUNC_OUT;
  return LIS_SUCCESS;
#else
  LIS_INT        err;
  LIS_INT        i,j,k,l,ii,jj,kk,ll;
  LIS_INT        n,nnz,annz,lfil,len;
  LIS_INT        cz,cw;
  LIS_INT        *iw,*wc,*wl,*iz,*zc,*zl;
  LIS_INT        *index,*ptr;
  LIS_SCALAR    gamma,val,t,tol,toldd;
  LIS_SCALAR    *z,*w,*tmp;
  LIS_SCALAR    *value;
  LIS_SCALAR    m;
  LIS_MATRIX    A;
  LIS_MATRIX_ILU  L,U;
  LIS_VECTOR    D;

  LIS_DEBUG_FUNC_IN;


  A      = solver->A;
  n      = A->n;
  tol    = solver->params[LIS_PARAMS_DROP-LIS_OPTIONS_LEN];
  m      = solver->params[LIS_PARAMS_RATE-LIS_OPTIONS_LEN];
  gamma  = solver->params[LIS_PARAMS_GAMMA-LIS_OPTIONS_LEN];
  annz   = 10+A->nnz / A->n;
  lfil   = (LIS_INT)(((double)A->nnz/(2.0*n))*m);

  L      = NULL;
  U      = NULL;


  err = lis_matrix_ilu_create(n,1,&L);
  if( err ) return err;
  err = lis_matrix_ilu_create(n,1,&U);
  if( err ) return err;
  err = lis_matrix_ilu_setCR(L);
  if( err ) return err;
  err = lis_matrix_ilu_setCR(U);
  if( err ) return err;
  err = lis_vector_duplicate(A,&D);
  if( err )
  {
    return err;
  }

  tmp   = (LIS_SCALAR *)lis_malloc(n*sizeof(LIS_SCALAR),"lis_precon_create_iluc_csr::tmp");
  if( tmp==NULL )
  {
    LIS_SETERR_MEM(n*sizeof(LIS_SCALAR));
    return LIS_OUT_OF_MEMORY;
  }
  w   = (LIS_SCALAR *)lis_malloc(n*sizeof(LIS_SCALAR),"lis_precon_create_iluc_csr::w");
  if( w==NULL )
  {
    LIS_SETERR_MEM(n*sizeof(LIS_SCALAR));
    return LIS_OUT_OF_MEMORY;
  }
  z   = (LIS_SCALAR *)lis_malloc(n*sizeof(LIS_SCALAR),"lis_precon_create_iluc_csr::z");
  if( z==NULL )
  {
    LIS_SETERR_MEM(n*sizeof(LIS_SCALAR));
    return LIS_OUT_OF_MEMORY;
  }

  iw   = (LIS_INT *)lis_malloc(n*sizeof(LIS_INT),"lis_precon_create_iluc_csr::iw");
  if( iw==NULL )
  {
    LIS_SETERR_MEM(n*sizeof(LIS_INT));
    return LIS_OUT_OF_MEMORY;
  }
  wc   = (LIS_INT *)lis_malloc(n*sizeof(LIS_INT),"lis_precon_create_iluc_csr::wc");
  if( wc==NULL )
  {
    LIS_SETERR_MEM(n*sizeof(LIS_INT));
    return LIS_OUT_OF_MEMORY;
  }
  wl   = (LIS_INT *)lis_malloc(n*sizeof(LIS_INT),"lis_precon_create_iluc_csr::wl");
  if( wl==NULL )
  {
    LIS_SETERR_MEM(n*sizeof(LIS_INT));
    return LIS_OUT_OF_MEMORY;
  }
  iz   = (LIS_INT *)lis_malloc(n*sizeof(LIS_INT),"lis_precon_create_iluc_csr::iz");
  if( iw==NULL )
  {
    LIS_SETERR_MEM(n*sizeof(LIS_INT));
    return LIS_OUT_OF_MEMORY;
  }
  zc   = (LIS_INT *)lis_malloc(n*sizeof(LIS_INT),"lis_precon_create_iluc_csr::zc");
  if( wc==NULL )
  {
    LIS_SETERR_MEM(n*sizeof(LIS_INT));
    return LIS_OUT_OF_MEMORY;
  }
  zl   = (LIS_INT *)lis_malloc(n*sizeof(LIS_INT),"lis_precon_create_iluc_csr::zl");
  if( wl==NULL )
  {
    LIS_SETERR_MEM(n*sizeof(LIS_INT));
    return LIS_OUT_OF_MEMORY;
  }

  lis_matrix_sort_csr(A);
  nnz = A->L->nnz;
  err = lis_matrix_malloc_csr(n,nnz,&ptr,&index,&value);

  for(i=0;i<n;i++) iw[i] = 0;
  for(i=0;i<n;i++)
  {
    for(j=A->L->ptr[i];j<A->L->ptr[i+1];j++)
    {
      jj = A->L->index[j];
      iw[jj]++;
    }
  }
  ptr[0] = 0;
  for(i=0;i<n;i++)
  {
    ptr[i+1] = ptr[i] + iw[i];
    iw[i]    = ptr[i];
  }
  for(i=0;i<n;i++)
  {
    for(j=A->L->ptr[i];j<A->L->ptr[i+1];j++)
    {
      k        = A->L->index[j];
      l        = iw[k];
      value[l] = A->L->value[j];
      index[l] = i;
      iw[k]++;
    }
  }


  for(i=0;i<n;i++)
  {
    D->value[i] = gamma * A->D->value[i];
    zl[i]       = -1;
    wl[i]       = -1;
    zc[i]       = 0;
    wc[i]       = 0;
  }


  for(k=0;k<n;k++)
  {
    cz = 0;
    cw = 0;
    /* z_j = a_kj (j>=k) */
    for(j=A->U->ptr[k];j<A->U->ptr[k+1];j++)
    {
      jj       = A->U->index[j];
      #ifdef USE_MPI
        if( jj>n-1 ) continue;
      #endif
      z[jj]    = A->U->value[j];
      iz[cz++] = jj;
      zc[jj]   = 1;
    }

    /* w_j = a_jk (j>k) */
    for(j=ptr[k];j<ptr[k+1];j++)
    {
      jj       = index[j];
      w[jj]    = value[j];
      iw[cw++] = jj;
      wc[jj]   = 1;
    }

    /* z_j = z_j - l_ki*u_ij */
    i = wl[k];
    while( i>=0 )
    {
      ii  = wc[i];
      kk  = zc[i];
      val = L->value[i][ii];
      for(j=kk;j<U->nnz[i];j++)
      {
        jj     = U->index[i][j];
        if( jj==k ) continue;
        if( zc[jj]==1 )
        {
          z[jj]   -= val*U->value[i][j];
        }
        else
        {
          z[jj]    = -val*U->value[i][j];
          iz[cz++] = jj;
          zc[jj]   = 1;
        }
      }
      ii++;
      wc[i] = ii;
      if( ii<L->nnz[i] )
      {
        kk     = L->index[i][ii];
        ll     = i;
        i      = wl[ll];
        wl[ll] = wl[kk];
        wl[kk] = ll;
      }
      else
      {
        i      = wl[i];
      }
    }

    /* w_j = w_j - l_ji*u_ik */
    i = zl[k];
    while( i>=0 )
    {
      ii  = zc[i];
      kk  = wc[i];
      val = U->value[i][ii];
      for(j=kk;j<L->nnz[i];j++)
      {
        jj     = L->index[i][j];
        if( wc[jj]==1 )
        {
          w[jj]   -= val*L->value[i][j];
        }
        else
        {
          w[jj]    = -val*L->value[i][j];
          iw[cw++] = jj;
          wc[jj]   = 1;
        }
      }
      ii++;
      zc[i] = ii;
      if( ii<U->nnz[i] )
      {
        kk     = U->index[i][ii];
        ll     = i;
        i      = zl[ll];
        zl[ll] = zl[kk];
        zl[kk] = ll;
      }
      else
      {
        i      = zl[i];
      }
    }

    toldd       = fabs(D->value[k])*tol;
    D->value[k] = 1.0/D->value[k];
    t           = D->value[k];
    if( cz<cw )
    {
      for(j=0;j<cz;j++)
      {
        jj = iz[j];
        if( wc[jj] )
        {
          D->value[jj] -= z[jj]*w[jj]*t;
        }
      }
    }
    else
    {
      for(j=0;j<cw;j++)
      {
        jj = iw[j];
        if( zc[jj] )
        {
          D->value[jj] -= z[jj]*w[jj]*t;
        }
      }
    }
    /* drop U */
    nnz = 0;
    for(j=0;j<cz;j++)
    {
      jj = iz[j];
      if( fabs(z[jj])>toldd )
      {
        iz[nnz++] = jj;
      }
      else
      {
        z[k] = z[k] + fabs(z[jj]);
        zc[jj] = 0;
      }
    }
    len = _min(lfil,nnz);
    for(j=0;j<nnz;j++) tmp[j] = fabs(z[j]);
    lis_sort_di(0,nnz-1,tmp,iz);
    lis_sort_i(0,len-1,iz);
    U->nnz[k] = len;
    if( len>0 )
    {
      U->index[k] = (LIS_INT *)malloc(len*sizeof(LIS_INT));
      U->value[k] = (LIS_SCALAR *)malloc(len*sizeof(LIS_SCALAR));
    }
    for(j=0;j<len;j++)
    {
      jj = iz[j];
      U->index[k][j] = jj;
      U->value[k][j] = z[jj];
    }
    for(j=len;j<cz;j++) zc[iz[j]] = 0;
    cz = nnz;

    /* drop L */
    nnz = 0;
    for(j=0;j<cw;j++)
    {
      jj = iw[j];
      if( fabs(w[jj])>toldd )
      {
        iw[nnz++] = jj;
      }
      else
      {
        wc[jj] = 0;
      }
    }
    len = _min(lfil,nnz);
    for(j=0;j<nnz;j++) tmp[j] = fabs(w[j]);
    lis_sort_di(0,nnz-1,tmp,iw);
    lis_sort_i(0,len-1,iw);
    L->nnz[k] = len;
    if( len>0 )
    {
      L->index[k] = (LIS_INT *)malloc(len*sizeof(LIS_INT));
      L->value[k] = (LIS_SCALAR *)malloc(len*sizeof(LIS_SCALAR));
    }
    for(j=0;j<len;j++)
    {
      jj = iw[j];
      L->index[k][j] = jj;
      L->value[k][j] = w[jj]*t;
    }
    for(j=len;j<cw;j++) wc[iw[j]] = 0;
    cw = nnz;

    /**/
    for(j=0;j<cz;j++) zc[iz[j]] = 0;
    for(j=0;j<cw;j++) wc[iw[j]] = 0;
    if(U->nnz[k]>0)
    {
      jj     = iz[0];
      zc[k]  = 0;
      zl[k]  = zl[jj];
      zl[jj] = k;
    }
    if(L->nnz[k]>0)
    {
      jj     = iw[0];
      wc[k]  = 0;
      wl[k]  = wl[jj];
      wl[jj] = k;
    }
  }

  precon->L  = L;
  precon->U  = U;
  precon->D  = D;

  lis_free2(12,tmp,w,z,iw,wc,wl,iz,zc,zl,ptr,index,value);

  LIS_DEBUG_FUNC_OUT;
  return LIS_SUCCESS;
#endif
}

#undef __FUNC__
#define __FUNC__ "lis_precon_create_iluc_bsr"
LIS_INT lis_precon_create_iluc_bsr(LIS_SOLVER solver, LIS_PRECON precon)
{
  LIS_INT        err;
  LIS_INT        i,j,k,l,ii,jj,kk,ll,bnr,bs;
  LIS_INT        n,nr,nnz,bnnz,annz,lfil,len;
  LIS_INT        cz,cw;
  LIS_INT        *iw,*wc,*wl,*iz,*zc,*zl;
  LIS_INT        *index,*ptr;
  LIS_SCALAR    gamma,val,t,tol,toldd;
  LIS_SCALAR    *z,*w,*tmp,wz[9],lu[9];
  LIS_SCALAR    *value;
  LIS_SCALAR    m,a;
  LIS_MATRIX    A;
  LIS_MATRIX_ILU  L,U;
  LIS_MATRIX_DIAG  D;

  LIS_DEBUG_FUNC_IN;


  A      = solver->A;
  n      = A->n;
  nr     = A->nr;
  bnr    = A->bnr;
  bs     = bnr*bnr;
  tol    = solver->params[LIS_PARAMS_DROP-LIS_OPTIONS_LEN];
  m      = solver->params[LIS_PARAMS_RATE-LIS_OPTIONS_LEN];
  gamma  = solver->params[LIS_PARAMS_GAMMA-LIS_OPTIONS_LEN];
  annz   = 10+A->bnnz / A->nr;
  lfil   = (LIS_INT)(((double)A->bnnz/(2.0*nr))*m);

  L      = NULL;
  U      = NULL;


  err = lis_matrix_ilu_create(nr,bnr,&L);
  if( err ) return err;
  err = lis_matrix_ilu_create(nr,bnr,&U);
  if( err ) return err;
  err = lis_matrix_ilu_setCR(L);
  if( err ) return err;
  err = lis_matrix_ilu_setCR(U);
  if( err ) return err;
  err = lis_matrix_diag_duplicateM(A,&D);
  if( err )
  {
    return err;
  }

  tmp   = (LIS_SCALAR *)lis_malloc(n*sizeof(LIS_SCALAR),"lis_precon_create_iluc_csr::tmp");
  if( tmp==NULL )
  {
    LIS_SETERR_MEM(n*sizeof(LIS_SCALAR));
    return LIS_OUT_OF_MEMORY;
  }
  w   = (LIS_SCALAR *)lis_malloc(bs*nr*sizeof(LIS_SCALAR),"lis_precon_create_iluc_csr::w");
  if( w==NULL )
  {
    LIS_SETERR_MEM(n*sizeof(LIS_SCALAR));
    return LIS_OUT_OF_MEMORY;
  }
  z   = (LIS_SCALAR *)lis_malloc(bs*nr*sizeof(LIS_SCALAR),"lis_precon_create_iluc_csr::z");
  if( z==NULL )
  {
    LIS_SETERR_MEM(n*sizeof(LIS_SCALAR));
    return LIS_OUT_OF_MEMORY;
  }

  iw   = (LIS_INT *)lis_malloc(n*sizeof(LIS_INT),"lis_precon_create_iluc_csr::iw");
  if( iw==NULL )
  {
    LIS_SETERR_MEM(n*sizeof(LIS_INT));
    return LIS_OUT_OF_MEMORY;
  }
  wc   = (LIS_INT *)lis_malloc(n*sizeof(LIS_INT),"lis_precon_create_iluc_csr::wc");
  if( wc==NULL )
  {
    LIS_SETERR_MEM(n*sizeof(LIS_INT));
    return LIS_OUT_OF_MEMORY;
  }
  wl   = (LIS_INT *)lis_malloc(n*sizeof(LIS_INT),"lis_precon_create_iluc_csr::wl");
  if( wl==NULL )
  {
    LIS_SETERR_MEM(n*sizeof(LIS_INT));
    return LIS_OUT_OF_MEMORY;
  }
  iz   = (LIS_INT *)lis_malloc(n*sizeof(LIS_INT),"lis_precon_create_iluc_csr::iz");
  if( iw==NULL )
  {
    LIS_SETERR_MEM(n*sizeof(LIS_INT));
    return LIS_OUT_OF_MEMORY;
  }
  zc   = (LIS_INT *)lis_malloc(n*sizeof(LIS_INT),"lis_precon_create_iluc_csr::zc");
  if( wc==NULL )
  {
    LIS_SETERR_MEM(n*sizeof(LIS_INT));
    return LIS_OUT_OF_MEMORY;
  }
  zl   = (LIS_INT *)lis_malloc(n*sizeof(LIS_INT),"lis_precon_create_iluc_csr::zl");
  if( wl==NULL )
  {
    LIS_SETERR_MEM(n*sizeof(LIS_INT));
    return LIS_OUT_OF_MEMORY;
  }

  bnnz = A->L->bnnz;
  err = lis_matrix_malloc_bsr(n,bnr,bnr,bnnz,&ptr,&index,&value);

  for(i=0;i<nr;i++) iw[i] = 0;
  for(i=0;i<nr;i++)
  {
    for(j=A->L->bptr[i];j<A->L->bptr[i+1];j++)
    {
      jj = A->L->bindex[j];
      iw[jj]++;
    }
  }
  ptr[0] = 0;
  for(i=0;i<nr;i++)
  {
    ptr[i+1] = ptr[i] + iw[i];
    iw[i]    = ptr[i];
  }
  for(i=0;i<nr;i++)
  {
    for(j=A->L->bptr[i];j<A->L->bptr[i+1];j++)
    {
      k        = A->L->bindex[j];
      l        = iw[k];
      memcpy(&value[bs*l],&A->L->value[bs*j],bs*sizeof(LIS_SCALAR));
      index[l] = i;
      iw[k]++;
    }
  }


  switch(bnr)
  {
  case 1:
    for(i=0;i<nr;i++)
    {
      D->value[i] = gamma*A->D->value[i];
      zl[i]       = -1;
      wl[i]       = -1;
      zc[i]       = 0;
      wc[i]       = 0;
    }
    break;
  case 2:
    for(i=0;i<nr;i++)
    {
      D->value[4*i+0] = gamma*A->D->value[4*i+0];
      D->value[4*i+1] = gamma*A->D->value[4*i+1];
      D->value[4*i+2] = gamma*A->D->value[4*i+2];
      D->value[4*i+3] = gamma*A->D->value[4*i+3];
      zl[i]       = -1;
      wl[i]       = -1;
      zc[i]       = 0;
      wc[i]       = 0;
    }
    if( n%2!=0 )
    {
      D->value[4*(nr-1)+3] = 1.0;
    }
    break;
  case 3:
    for(i=0;i<nr;i++)
    {
      D->value[9*i+0] = gamma*A->D->value[9*i+0];
      D->value[9*i+1] = gamma*A->D->value[9*i+1];
      D->value[9*i+2] = gamma*A->D->value[9*i+2];
      D->value[9*i+3] = gamma*A->D->value[9*i+3];
      D->value[9*i+4] = gamma*A->D->value[9*i+4];
      D->value[9*i+5] = gamma*A->D->value[9*i+5];
      D->value[9*i+6] = gamma*A->D->value[9*i+6];
      D->value[9*i+7] = gamma*A->D->value[9*i+7];
      D->value[9*i+8] = gamma*A->D->value[9*i+8];
      zl[i]       = -1;
      wl[i]       = -1;
      zc[i]       = 0;
      wc[i]       = 0;
    }
      if( n%3==1 )
      {
        D->value[9*(nr-1)+4] = 1.0;
        D->value[9*(nr-1)+8] = 1.0;
      }
      else if( n%3==2 )
      {
        D->value[9*(nr-1)+8] = 1.0;
      }
    break;
  }


  for(k=0;k<nr;k++)
  {
    cz = 0;
    cw = 0;
    /* z_j = a_kj (j>=k) */
    for(j=A->U->bptr[k];j<A->U->bptr[k+1];j++)
    {
      jj       = A->U->bindex[j];
      #ifdef USE_MPI
        if( jj>nr-1 ) continue;
      #endif
#if 0
      switch(bnr)
      {
      case 1:
        toldd       = fabs(D->value[k]);
        t           = fabs(A->U->value[j]);
        break;
      case 2:
        toldd       = fabs(D->value[4*k+0]) + fabs(D->value[4*k+1]);
        toldd      += fabs(D->value[4*k+2]) + fabs(D->value[4*k+3]);
/*        toldd       = sqrt(toldd);*/
        t       = fabs(A->U->value[4*j+0]) + fabs(A->U->value[4*j+1]);
        t      += fabs(A->U->value[4*j+2]) + fabs(A->U->value[4*j+3]);
/*        t       = sqrt(t);*/
        break;
      case 3:
        toldd       = fabs(D->value[9*k+0]) + fabs(D->value[9*k+1]) + fabs(D->value[9*k+2]);
        toldd      += fabs(D->value[9*k+3]) + fabs(D->value[9*k+4]) + fabs(D->value[9*k+5]);
        toldd      += fabs(D->value[9*k+6]) + fabs(D->value[9*k+7]) + fabs(D->value[9*k+8]);
/*        toldd       = sqrt(toldd);*/
        t       = fabs(A->U->value[9*j+0]) + fabs(A->U->value[9*j+1]) + fabs(A->U->value[9*j+2]);
        t      += fabs(A->U->value[9*j+3]) + fabs(A->U->value[9*j+4]) + fabs(A->U->value[9*j+5]);
        t      += fabs(A->U->value[9*j+6]) + fabs(A->U->value[9*j+7]) + fabs(A->U->value[9*j+8]);
/*        t       = sqrt(t);*/
        break;
      }
      if( toldd>t*1.0e-10 ) 
      {
        memcpy(&z[bs*jj],&A->U->value[bs*j],bs*sizeof(LIS_SCALAR));
        iz[cz++] = jj;
        zc[jj]   = 1;
      }
#else
      memcpy(&z[bs*jj],&A->U->value[bs*j],bs*sizeof(LIS_SCALAR));
      iz[cz++] = jj;
      zc[jj]   = 1;
#endif
    }

    /* w_j = a_jk (j>k) */
    for(j=ptr[k];j<ptr[k+1];j++)
    {
      jj       = index[j];
#if 0
      switch(bnr)
      {
      case 1:
        toldd       = fabs(D->value[k]);
        t           = fabs(value[j]);
        break;
      case 2:
        toldd       = fabs(D->value[4*k+0]) + fabs(D->value[4*k+1]);
        toldd      += fabs(D->value[4*k+2]) + fabs(D->value[4*k+3]);
/*        toldd       = sqrt(toldd);*/
        t       = fabs(value[4*j+0]) + fabs(value[4*j+1]);
        t      += fabs(value[4*j+2]) + fabs(value[4*j+3]);
/*        t       = sqrt(t);*/
        break;
      case 3:
        toldd       = fabs(D->value[9*k+0]) + fabs(D->value[9*k+1]) + fabs(D->value[9*k+2]);
        toldd      += fabs(D->value[9*k+3]) + fabs(D->value[9*k+4]) + fabs(D->value[9*k+5]);
        toldd      += fabs(D->value[9*k+6]) + fabs(D->value[9*k+7]) + fabs(D->value[9*k+8]);
/*        toldd       = sqrt(toldd);*/
        t       = fabs(value[9*j+0]) + fabs(value[9*j+1]) + fabs(value[9*j+2]);
        t      += fabs(value[9*j+3]) + fabs(value[9*j+4]) + fabs(value[9*j+5]);
        t      += fabs(value[9*j+6]) + fabs(value[9*j+7]) + fabs(value[9*j+8]);
/*        t       = sqrt(t);*/
        break;
      }
      if( toldd>t*1.0e-10 ) 
      {
        memcpy(&w[bs*jj],&value[bs*j],bs*sizeof(LIS_SCALAR));
        iw[cw++] = jj;
        wc[jj]   = 1;
      }
#else
      memcpy(&w[bs*jj],&value[bs*j],bs*sizeof(LIS_SCALAR));
      iw[cw++] = jj;
      wc[jj]   = 1;
#endif
    }

    /* z_j = z_j - l_ki*u_ij */
    i = wl[k];
    while( i>=0 )
    {
      ii  = wc[i];
      kk  = zc[i];
      val = L->value[i][ii];
      for(j=kk;j<U->nnz[i];j++)
      {
        jj     = U->index[i][j];
#if 1
        if( jj==k ) continue;
        if( zc[jj]==1 )
#else
        if( jj==k )
        {
          switch(bnr)
          {
          case 1:
            D->value[jj]   -= val*U->value[i][j];
            break;
          case 2:
            D->value[4*jj+0] -= L->value[i][4*ii+0]*U->value[i][4*j+0];
            D->value[4*jj+0] -= L->value[i][4*ii+2]*U->value[i][4*j+1];
            D->value[4*jj+1] -= L->value[i][4*ii+1]*U->value[i][4*j+0];
            D->value[4*jj+1] -= L->value[i][4*ii+3]*U->value[i][4*j+1];
            D->value[4*jj+2] -= L->value[i][4*ii+0]*U->value[i][4*j+2];
            D->value[4*jj+2] -= L->value[i][4*ii+2]*U->value[i][4*j+3];
            D->value[4*jj+3] -= L->value[i][4*ii+1]*U->value[i][4*j+2];
            D->value[4*jj+3] -= L->value[i][4*ii+3]*U->value[i][4*j+3];
            break;
          case 3:
            D->value[9*jj+0] -= L->value[i][9*ii+0]*U->value[i][9*j+0] + L->value[i][9*ii+3]*U->value[i][9*j+1] + L->value[i][9*ii+6]*U->value[i][9*j+2];
            D->value[9*jj+1] -= L->value[i][9*ii+1]*U->value[i][9*j+0] + L->value[i][9*ii+4]*U->value[i][9*j+1] + L->value[i][9*ii+7]*U->value[i][9*j+2];
            D->value[9*jj+2] -= L->value[i][9*ii+2]*U->value[i][9*j+0] + L->value[i][9*ii+5]*U->value[i][9*j+1] + L->value[i][9*ii+8]*U->value[i][9*j+2];
            D->value[9*jj+3] -= L->value[i][9*ii+0]*U->value[i][9*j+3] + L->value[i][9*ii+3]*U->value[i][9*j+4] + L->value[i][9*ii+6]*U->value[i][9*j+5];
            D->value[9*jj+4] -= L->value[i][9*ii+1]*U->value[i][9*j+3] + L->value[i][9*ii+4]*U->value[i][9*j+4] + L->value[i][9*ii+7]*U->value[i][9*j+5];
            D->value[9*jj+5] -= L->value[i][9*ii+2]*U->value[i][9*j+3] + L->value[i][9*ii+5]*U->value[i][9*j+4] + L->value[i][9*ii+8]*U->value[i][9*j+5];
            D->value[9*jj+6] -= L->value[i][9*ii+0]*U->value[i][9*j+6] + L->value[i][9*ii+3]*U->value[i][9*j+7] + L->value[i][9*ii+6]*U->value[i][9*j+8];
            D->value[9*jj+7] -= L->value[i][9*ii+1]*U->value[i][9*j+6] + L->value[i][9*ii+4]*U->value[i][9*j+7] + L->value[i][9*ii+7]*U->value[i][9*j+8];
            D->value[9*jj+8] -= L->value[i][9*ii+2]*U->value[i][9*j+6] + L->value[i][9*ii+5]*U->value[i][9*j+7] + L->value[i][9*ii+8]*U->value[i][9*j+8];
            break;
          }
        }
        else if( zc[jj]==1 )
#endif
        {
          switch(bnr)
          {
          case 1:
            z[jj]   -= val*U->value[i][j];
            break;
          case 2:
            z[4*jj+0] -= L->value[i][4*ii+0]*U->value[i][4*j+0];
            z[4*jj+0] -= L->value[i][4*ii+2]*U->value[i][4*j+1];
            z[4*jj+1] -= L->value[i][4*ii+1]*U->value[i][4*j+0];
            z[4*jj+1] -= L->value[i][4*ii+3]*U->value[i][4*j+1];
            z[4*jj+2] -= L->value[i][4*ii+0]*U->value[i][4*j+2];
            z[4*jj+2] -= L->value[i][4*ii+2]*U->value[i][4*j+3];
            z[4*jj+3] -= L->value[i][4*ii+1]*U->value[i][4*j+2];
            z[4*jj+3] -= L->value[i][4*ii+3]*U->value[i][4*j+3];
            break;
          case 3:
            z[9*jj+0] -= L->value[i][9*ii+0]*U->value[i][9*j+0] + L->value[i][9*ii+3]*U->value[i][9*j+1] + L->value[i][9*ii+6]*U->value[i][9*j+2];
            z[9*jj+1] -= L->value[i][9*ii+1]*U->value[i][9*j+0] + L->value[i][9*ii+4]*U->value[i][9*j+1] + L->value[i][9*ii+7]*U->value[i][9*j+2];
            z[9*jj+2] -= L->value[i][9*ii+2]*U->value[i][9*j+0] + L->value[i][9*ii+5]*U->value[i][9*j+1] + L->value[i][9*ii+8]*U->value[i][9*j+2];
            z[9*jj+3] -= L->value[i][9*ii+0]*U->value[i][9*j+3] + L->value[i][9*ii+3]*U->value[i][9*j+4] + L->value[i][9*ii+6]*U->value[i][9*j+5];
            z[9*jj+4] -= L->value[i][9*ii+1]*U->value[i][9*j+3] + L->value[i][9*ii+4]*U->value[i][9*j+4] + L->value[i][9*ii+7]*U->value[i][9*j+5];
            z[9*jj+5] -= L->value[i][9*ii+2]*U->value[i][9*j+3] + L->value[i][9*ii+5]*U->value[i][9*j+4] + L->value[i][9*ii+8]*U->value[i][9*j+5];
            z[9*jj+6] -= L->value[i][9*ii+0]*U->value[i][9*j+6] + L->value[i][9*ii+3]*U->value[i][9*j+7] + L->value[i][9*ii+6]*U->value[i][9*j+8];
            z[9*jj+7] -= L->value[i][9*ii+1]*U->value[i][9*j+6] + L->value[i][9*ii+4]*U->value[i][9*j+7] + L->value[i][9*ii+7]*U->value[i][9*j+8];
            z[9*jj+8] -= L->value[i][9*ii+2]*U->value[i][9*j+6] + L->value[i][9*ii+5]*U->value[i][9*j+7] + L->value[i][9*ii+8]*U->value[i][9*j+8];
            break;
          }
        }
        else
        {
          switch(bnr)
          {
          case 1:
            z[jj]   = -val*U->value[i][j];
            break;
          case 2:
            z[4*jj+0] = -L->value[i][4*ii+0]*U->value[i][4*j+0] - L->value[i][4*ii+2]*U->value[i][4*j+1];
            z[4*jj+1] = -L->value[i][4*ii+1]*U->value[i][4*j+0] - L->value[i][4*ii+3]*U->value[i][4*j+1];
            z[4*jj+2] = -L->value[i][4*ii+0]*U->value[i][4*j+2] - L->value[i][4*ii+2]*U->value[i][4*j+3];
            z[4*jj+3] = -L->value[i][4*ii+1]*U->value[i][4*j+2] - L->value[i][4*ii+3]*U->value[i][4*j+3];
            break;
          case 3:
            z[9*jj+0] = -L->value[i][9*ii+0]*U->value[i][9*j+0] - L->value[i][9*ii+3]*U->value[i][9*j+1] - L->value[i][9*ii+6]*U->value[i][9*j+2];
            z[9*jj+1] = -L->value[i][9*ii+1]*U->value[i][9*j+0] - L->value[i][9*ii+4]*U->value[i][9*j+1] - L->value[i][9*ii+7]*U->value[i][9*j+2];
            z[9*jj+2] = -L->value[i][9*ii+2]*U->value[i][9*j+0] - L->value[i][9*ii+5]*U->value[i][9*j+1] - L->value[i][9*ii+8]*U->value[i][9*j+2];
            z[9*jj+3] = -L->value[i][9*ii+0]*U->value[i][9*j+3] - L->value[i][9*ii+3]*U->value[i][9*j+4] - L->value[i][9*ii+6]*U->value[i][9*j+5];
            z[9*jj+4] = -L->value[i][9*ii+1]*U->value[i][9*j+3] - L->value[i][9*ii+4]*U->value[i][9*j+4] - L->value[i][9*ii+7]*U->value[i][9*j+5];
            z[9*jj+5] = -L->value[i][9*ii+2]*U->value[i][9*j+3] - L->value[i][9*ii+5]*U->value[i][9*j+4] - L->value[i][9*ii+8]*U->value[i][9*j+5];
            z[9*jj+6] = -L->value[i][9*ii+0]*U->value[i][9*j+6] - L->value[i][9*ii+3]*U->value[i][9*j+7] - L->value[i][9*ii+6]*U->value[i][9*j+8];
            z[9*jj+7] = -L->value[i][9*ii+1]*U->value[i][9*j+6] - L->value[i][9*ii+4]*U->value[i][9*j+7] - L->value[i][9*ii+7]*U->value[i][9*j+8];
            z[9*jj+8] = -L->value[i][9*ii+2]*U->value[i][9*j+6] - L->value[i][9*ii+5]*U->value[i][9*j+7] - L->value[i][9*ii+8]*U->value[i][9*j+8];
            break;
          }
          iz[cz++] = jj;
          zc[jj]   = 1;
        }
      }
      ii++;
      wc[i] = ii;
      if( ii<L->nnz[i] )
      {
        kk     = L->index[i][ii];
        ll     = i;
        i      = wl[ll];
        wl[ll] = wl[kk];
        wl[kk] = ll;
      }
      else
      {
        i      = wl[i];
      }
    }

    /* w_j = w_j - l_ji*u_ik */
    i = zl[k];
    while( i>=0 )
    {
      ii  = zc[i];
      kk  = wc[i];
      val = U->value[i][ii];
      for(j=kk;j<L->nnz[i];j++)
      {
        jj     = L->index[i][j];
        if( wc[jj]==1 )
        {
          switch(bnr)
          {
          case 1:
            w[jj]   -= val*L->value[i][j];
            break;
          case 2:
            w[4*jj+0] -= U->value[i][4*ii+0]*L->value[i][4*j+0];
            w[4*jj+0] -= U->value[i][4*ii+2]*L->value[i][4*j+1];
            w[4*jj+1] -= U->value[i][4*ii+1]*L->value[i][4*j+0];
            w[4*jj+1] -= U->value[i][4*ii+3]*L->value[i][4*j+1];
            w[4*jj+2] -= U->value[i][4*ii+0]*L->value[i][4*j+2];
            w[4*jj+2] -= U->value[i][4*ii+2]*L->value[i][4*j+3];
            w[4*jj+3] -= U->value[i][4*ii+1]*L->value[i][4*j+2];
            w[4*jj+3] -= U->value[i][4*ii+3]*L->value[i][4*j+3];

/*            w[4*jj+0] -= L->value[i][4*j+0]*U->value[i][4*ii+0];
            w[4*jj+0] -= L->value[i][4*j+2]*U->value[i][4*ii+1];
            w[4*jj+1] -= L->value[i][4*j+1]*U->value[i][4*ii+0];
            w[4*jj+1] -= L->value[i][4*j+3]*U->value[i][4*ii+1];
            w[4*jj+2] -= L->value[i][4*j+0]*U->value[i][4*ii+2];
            w[4*jj+2] -= L->value[i][4*j+2]*U->value[i][4*ii+3];
            w[4*jj+3] -= L->value[i][4*j+1]*U->value[i][4*ii+2];
            w[4*jj+3] -= L->value[i][4*j+3]*U->value[i][4*ii+3];*/
            break;
          case 3:
            w[9*jj+0] -= L->value[i][9*j+0]*U->value[i][9*ii+0] + L->value[i][9*j+3]*U->value[i][9*ii+1] + L->value[i][9*j+6]*U->value[i][9*ii+2];
            w[9*jj+1] -= L->value[i][9*j+1]*U->value[i][9*ii+0] + L->value[i][9*j+4]*U->value[i][9*ii+1] + L->value[i][9*j+7]*U->value[i][9*ii+2];
            w[9*jj+2] -= L->value[i][9*j+2]*U->value[i][9*ii+0] + L->value[i][9*j+5]*U->value[i][9*ii+1] + L->value[i][9*j+8]*U->value[i][9*ii+2];
            w[9*jj+3] -= L->value[i][9*j+0]*U->value[i][9*ii+3] + L->value[i][9*j+3]*U->value[i][9*ii+4] + L->value[i][9*j+6]*U->value[i][9*ii+5];
            w[9*jj+4] -= L->value[i][9*j+1]*U->value[i][9*ii+3] + L->value[i][9*j+4]*U->value[i][9*ii+4] + L->value[i][9*j+7]*U->value[i][9*ii+5];
            w[9*jj+5] -= L->value[i][9*j+2]*U->value[i][9*ii+3] + L->value[i][9*j+5]*U->value[i][9*ii+4] + L->value[i][9*j+8]*U->value[i][9*ii+5];
            w[9*jj+6] -= L->value[i][9*j+0]*U->value[i][9*ii+6] + L->value[i][9*j+3]*U->value[i][9*ii+7] + L->value[i][9*j+6]*U->value[i][9*ii+8];
            w[9*jj+7] -= L->value[i][9*j+1]*U->value[i][9*ii+6] + L->value[i][9*j+4]*U->value[i][9*ii+7] + L->value[i][9*j+7]*U->value[i][9*ii+8];
            w[9*jj+8] -= L->value[i][9*j+2]*U->value[i][9*ii+6] + L->value[i][9*j+5]*U->value[i][9*ii+7] + L->value[i][9*j+8]*U->value[i][9*ii+8];

/*            w[9*jj+0] -= U->value[i][9*ii+0]*L->value[i][9*j+0] + U->value[i][9*ii+3]*L->value[i][9*j+1] + U->value[i][9*ii+6]*L->value[i][9*j+2];
            w[9*jj+1] -= U->value[i][9*ii+1]*L->value[i][9*j+0] + U->value[i][9*ii+4]*L->value[i][9*j+1] + U->value[i][9*ii+7]*L->value[i][9*j+2];
            w[9*jj+2] -= U->value[i][9*ii+2]*L->value[i][9*j+0] + U->value[i][9*ii+5]*L->value[i][9*j+1] + U->value[i][9*ii+8]*L->value[i][9*j+2];
            w[9*jj+3] -= U->value[i][9*ii+0]*L->value[i][9*j+3] + U->value[i][9*ii+3]*L->value[i][9*j+4] + U->value[i][9*ii+6]*L->value[i][9*j+5];
            w[9*jj+4] -= U->value[i][9*ii+1]*L->value[i][9*j+3] + U->value[i][9*ii+4]*L->value[i][9*j+4] + U->value[i][9*ii+7]*L->value[i][9*j+5];
            w[9*jj+5] -= U->value[i][9*ii+2]*L->value[i][9*j+3] + U->value[i][9*ii+5]*L->value[i][9*j+4] + U->value[i][9*ii+8]*L->value[i][9*j+5];
            w[9*jj+6] -= U->value[i][9*ii+0]*L->value[i][9*j+6] + U->value[i][9*ii+3]*L->value[i][9*j+7] + U->value[i][9*ii+6]*L->value[i][9*j+8];
            w[9*jj+7] -= U->value[i][9*ii+1]*L->value[i][9*j+6] + U->value[i][9*ii+4]*L->value[i][9*j+7] + U->value[i][9*ii+7]*L->value[i][9*j+8];
            w[9*jj+8] -= U->value[i][9*ii+2]*L->value[i][9*j+6] + U->value[i][9*ii+5]*L->value[i][9*j+7] + U->value[i][9*ii+8]*L->value[i][9*j+8];*/
            break;
          }
        }
        else
        {
          switch(bnr)
          {
          case 1:
            w[jj]   = -val*L->value[i][j];
            break;
          case 2:
            w[4*jj+0] = -U->value[i][4*ii+0]*L->value[i][4*j+0] - U->value[i][4*ii+2]*L->value[i][4*j+1];
            w[4*jj+1] = -U->value[i][4*ii+1]*L->value[i][4*j+0] - U->value[i][4*ii+3]*L->value[i][4*j+1];
            w[4*jj+2] = -U->value[i][4*ii+0]*L->value[i][4*j+2] - U->value[i][4*ii+2]*L->value[i][4*j+3];
            w[4*jj+3] = -U->value[i][4*ii+1]*L->value[i][4*j+2] - U->value[i][4*ii+3]*L->value[i][4*j+3];

/*            w[4*jj+0] = -L->value[i][4*j+0]*U->value[i][4*ii+0] - L->value[i][4*j+2]*U->value[i][4*ii+1];
            w[4*jj+1] = -L->value[i][4*j+1]*U->value[i][4*ii+0] - L->value[i][4*j+3]*U->value[i][4*ii+1];
            w[4*jj+2] = -L->value[i][4*j+0]*U->value[i][4*ii+2] - L->value[i][4*j+2]*U->value[i][4*ii+3];
            w[4*jj+3] = -L->value[i][4*j+1]*U->value[i][4*ii+2] - L->value[i][4*j+3]*U->value[i][4*ii+3];*/
            break;
          case 3:
            w[9*jj+0] = -L->value[i][9*j+0]*U->value[i][9*ii+0] - L->value[i][9*j+3]*U->value[i][9*ii+1] - L->value[i][9*j+6]*U->value[i][9*ii+2];
            w[9*jj+1] = -L->value[i][9*j+1]*U->value[i][9*ii+0] - L->value[i][9*j+4]*U->value[i][9*ii+1] - L->value[i][9*j+7]*U->value[i][9*ii+2];
            w[9*jj+2] = -L->value[i][9*j+2]*U->value[i][9*ii+0] - L->value[i][9*j+5]*U->value[i][9*ii+1] - L->value[i][9*j+8]*U->value[i][9*ii+2];
            w[9*jj+3] = -L->value[i][9*j+0]*U->value[i][9*ii+3] - L->value[i][9*j+3]*U->value[i][9*ii+4] - L->value[i][9*j+6]*U->value[i][9*ii+5];
            w[9*jj+4] = -L->value[i][9*j+1]*U->value[i][9*ii+3] - L->value[i][9*j+4]*U->value[i][9*ii+4] - L->value[i][9*j+7]*U->value[i][9*ii+5];
            w[9*jj+5] = -L->value[i][9*j+2]*U->value[i][9*ii+3] - L->value[i][9*j+5]*U->value[i][9*ii+4] - L->value[i][9*j+8]*U->value[i][9*ii+5];
            w[9*jj+6] = -L->value[i][9*j+0]*U->value[i][9*ii+6] - L->value[i][9*j+3]*U->value[i][9*ii+7] - L->value[i][9*j+6]*U->value[i][9*ii+8];
            w[9*jj+7] = -L->value[i][9*j+1]*U->value[i][9*ii+6] - L->value[i][9*j+4]*U->value[i][9*ii+7] - L->value[i][9*j+7]*U->value[i][9*ii+8];
            w[9*jj+8] = -L->value[i][9*j+2]*U->value[i][9*ii+6] - L->value[i][9*j+5]*U->value[i][9*ii+7] - L->value[i][9*j+8]*U->value[i][9*ii+8];

/*            w[9*jj+0] = -U->value[i][9*ii+0]*L->value[i][9*j+0] - U->value[i][9*ii+3]*L->value[i][9*j+1] - U->value[i][9*ii+6]*L->value[i][9*j+2];
            w[9*jj+1] = -U->value[i][9*ii+1]*L->value[i][9*j+0] - U->value[i][9*ii+4]*L->value[i][9*j+1] - U->value[i][9*ii+7]*L->value[i][9*j+2];
            w[9*jj+2] = -U->value[i][9*ii+2]*L->value[i][9*j+0] - U->value[i][9*ii+5]*L->value[i][9*j+1] - U->value[i][9*ii+8]*L->value[i][9*j+2];
            w[9*jj+3] = -U->value[i][9*ii+0]*L->value[i][9*j+3] - U->value[i][9*ii+3]*L->value[i][9*j+4] - U->value[i][9*ii+6]*L->value[i][9*j+5];
            w[9*jj+4] = -U->value[i][9*ii+1]*L->value[i][9*j+3] - U->value[i][9*ii+4]*L->value[i][9*j+4] - U->value[i][9*ii+7]*L->value[i][9*j+5];
            w[9*jj+5] = -U->value[i][9*ii+2]*L->value[i][9*j+3] - U->value[i][9*ii+5]*L->value[i][9*j+4] - U->value[i][9*ii+8]*L->value[i][9*j+5];
            w[9*jj+6] = -U->value[i][9*ii+0]*L->value[i][9*j+6] - U->value[i][9*ii+3]*L->value[i][9*j+7] - U->value[i][9*ii+6]*L->value[i][9*j+8];
            w[9*jj+7] = -U->value[i][9*ii+1]*L->value[i][9*j+6] - U->value[i][9*ii+4]*L->value[i][9*j+7] - U->value[i][9*ii+7]*L->value[i][9*j+8];
            w[9*jj+8] = -U->value[i][9*ii+2]*L->value[i][9*j+6] - U->value[i][9*ii+5]*L->value[i][9*j+7] - U->value[i][9*ii+8]*L->value[i][9*j+8];*/
            break;
          }
          iw[cw++] = jj;
          wc[jj]   = 1;
        }
      }
      ii++;
      zc[i] = ii;
      if( ii<U->nnz[i] )
      {
        kk     = U->index[i][ii];
        ll     = i;
        i      = zl[ll];
        zl[ll] = zl[kk];
        zl[kk] = ll;
      }
      else
      {
        i      = zl[i];
      }
    }

    switch(bnr)
    {
    case 1:
      toldd       = fabs(D->value[k])*tol;
      break;
    case 2:
      toldd       = fabs(D->value[4*k+0]) + fabs(D->value[4*k+1]);
      toldd      += fabs(D->value[4*k+2]) + fabs(D->value[4*k+3]);
/*      toldd       = sqrt(toldd)*tol;*/
      toldd       = toldd*tol;
      break;
    case 3:
      toldd       = fabs(D->value[9*k+0]) + fabs(D->value[9*k+1]) + fabs(D->value[9*k+2]);
      toldd      += fabs(D->value[9*k+3]) + fabs(D->value[9*k+4]) + fabs(D->value[9*k+5]);
      toldd      += fabs(D->value[9*k+6]) + fabs(D->value[9*k+7]) + fabs(D->value[9*k+8]);
/*      toldd       = sqrt(toldd)*tol;*/
      toldd       = toldd*tol;
      break;
    }
/*    D->value[k] = 1.0/D->value[k];*/
/*    t           = D->value[k];*/

#if 1
    memcpy(lu,&D->value[bs*k],bs*sizeof(LIS_SCALAR));
    for(kk=0;kk<bnr;kk++)
    {
      lu[kk*bnr+kk] = 1.0 / lu[kk*bnr+kk];
      for(ii=kk+1;ii<bnr;ii++)
      {
        a = lu[kk*bnr+ii] * lu[kk*bnr+kk];
        for(jj=kk+1;jj<bnr;jj++)
        {
          lu[jj*bnr+ii] -= a * lu[jj*bnr+kk];
        }
        lu[kk*bnr+ii] = a;
      }
    }
    memcpy(&D->value[bs*k],lu,bs*sizeof(LIS_SCALAR));
    switch(bnr)
#ifdef _LONGLONG
    {
    case 1:
      printf("k=%lld toldd=%e\n",k,toldd);
      printf("k=%lld %e\n",k,D->value[k]);
      break;
    case 2:
      printf("k=%lld toldd=%e\n",k,toldd);
      printf("k=%lld %e %e\n",k,D->value[4*k+0],D->value[4*k+2]);
      printf("k=%lld %e %e\n",k,D->value[4*k+1],D->value[4*k+3]);
      break;
    case 3:
      printf("k=%lld toldd=%e\n",k,toldd);
      printf("k=%lld %e %e %e\n",k,D->value[9*k+0],D->value[9*k+3],D->value[9*k+6]);
      printf("k=%lld %e %e %e\n",k,D->value[9*k+1],D->value[9*k+4],D->value[9*k+7]);
      printf("k=%lld %e %e %e\n",k,D->value[9*k+2],D->value[9*k+5],D->value[9*k+8]);
      break;
    }
#else
    {
    case 1:
      printf("k=%d toldd=%e\n",k,toldd);
      printf("k=%d %e\n",k,D->value[k]);
      break;
    case 2:
      printf("k=%d toldd=%e\n",k,toldd);
      printf("k=%d %e %e\n",k,D->value[4*k+0],D->value[4*k+2]);
      printf("k=%d %e %e\n",k,D->value[4*k+1],D->value[4*k+3]);
      break;
    case 3:
      printf("k=%d toldd=%e\n",k,toldd);
      printf("k=%d %e %e %e\n",k,D->value[9*k+0],D->value[9*k+3],D->value[9*k+6]);
      printf("k=%d %e %e %e\n",k,D->value[9*k+1],D->value[9*k+4],D->value[9*k+7]);
      printf("k=%d %e %e %e\n",k,D->value[9*k+2],D->value[9*k+5],D->value[9*k+8]);
      break;
    }
#endif
#endif
#if 0
    switch(bnr)
    {
    case 1:
      D->value[k] = 1.0/D->value[k];
      break;
    case 2:
      /*
        D[i] = |d0 d2|
               |d1 d3|
        */
      t        = 1.0/(D->value[4*k]*D->value[4*k+3] - D->value[4*k+1]*D->value[4*k+2]);

      a        = D->value[4*k];
      printf("k=%d t=%e\n",k,t);
      printf("k=%d %e %e\n",k,D->value[4*k+0],D->value[4*k+2]);
      printf("k=%d %e %e\n",k,D->value[4*k+1],D->value[4*k+3]);
      D->value[4*k]   = t*D->value[4*k+3];
      D->value[4*k+1] = -t*D->value[4*k+1];
      D->value[4*k+2] = -t*D->value[4*k+2];
      D->value[4*k+3] = t*a;
      break;
    case 3:
      /*
        D[i] = |d0 d3 d6|
            |d1 d4 d7|
          |d2 d5 d8|
        */
      t       = D->value[9*k]*D->value[9*k+4]*D->value[9*k+8] + D->value[9*k+1]*D->value[9*k+5]*D->value[9*k+6] + D->value[9*k+2]*D->value[9*k+3]*D->value[9*k+7];
      t      -= D->value[9*k]*D->value[9*k+5]*D->value[9*k+7] + D->value[9*k+1]*D->value[9*k+3]*D->value[9*k+8] + D->value[9*k+2]*D->value[9*k+4]*D->value[9*k+6];
      printf("k=%d t=%e\n",k,t);
      t       = 1.0 / t;
      b[0]    = t*(D->value[9*k+4]*D->value[9*k+8] - D->value[9*k+5]*D->value[9*k+7]);
      b[1]    = t*(D->value[9*k+5]*D->value[9*k+6] - D->value[9*k+3]*D->value[9*k+8]);
      b[2]    = t*(D->value[9*k+3]*D->value[9*k+7] - D->value[9*k+4]*D->value[9*k+6]);
      b[3]    = t*(D->value[9*k+2]*D->value[9*k+7] - D->value[9*k+1]*D->value[9*k+8]);
      b[4]    = t*(D->value[9*k+0]*D->value[9*k+8] - D->value[9*k+2]*D->value[9*k+6]);
      b[5]    = t*(D->value[9*k+1]*D->value[9*k+6] - D->value[9*k+0]*D->value[9*k+7]);
      b[6]    = t*(D->value[9*k+1]*D->value[9*k+5] - D->value[9*k+2]*D->value[9*k+4]);
      b[7]    = t*(D->value[9*k+2]*D->value[9*k+3] - D->value[9*k+0]*D->value[9*k+5]);
      b[8]    = t*(D->value[9*k+0]*D->value[9*k+4] - D->value[9*k+1]*D->value[9*k+3]);
      memcpy(&D->value[9*k],b,9*sizeof(LIS_SCALAR));
/*      printf("k=%d t=%e\n",k,t);
      printf("k=%d %e %e %e\n",k,b[0],b[3],b[6]);
      printf("k=%d %e %e %e\n",k,b[1],b[4],b[7]);
      printf("k=%d %e %e %e\n",k,b[2],b[5],b[8]);*/
      break;
    }
    memcpy(tt,&D->value[bs*k],bs*sizeof(LIS_SCALAR));
#endif
#if 1
    if( cz<cw )
    {
      for(j=0;j<cz;j++)
      {
        jj = iz[j];
        if( wc[jj] )
        {
#if 1
          for(i=0;i<bnr;i++)
          {
            wz[i] = -w[bs*jj+i]*lu[0];
            for(ii=1;ii<bnr;ii++)
            {
              a = -w[bs*jj+ii*bnr+i];
              for(kk=0;kk<ii-1;kk++)
              {
                a -= wz[kk*bnr+i] * lu[ii*bnr+kk];
              }
              wz[ii*bnr+i] = a * lu[ii*bnr+ii];
            }
          }
          for(i=0;i<bnr;i++)
          {
            for(ii=bnr-1;ii>=0;ii--)
            {
              a = wz[ii*bnr+i];
              for(kk=ii+1;kk<bnr;kk++)
              {
                a -= wz[kk*bnr+i] * lu[ii*bnr+kk];
              }
              wz[ii*bnr+i] = a;
            }
          }
#endif
          switch(bnr)
          {
          case 1:
/*            D->value[jj] -= z[jj]*w[jj];*/
/*            D->value[jj] -= z[jj]*w[jj]*tt[0];*/
            D->value[jj] -= z[jj]*wz[0];
            break;
          case 2:
/*            wz[0] = w[4*jj+0]*tt[0] + w[4*jj+2]*tt[1];
            wz[1] = w[4*jj+1]*tt[0] + w[4*jj+3]*tt[1];
            wz[2] = w[4*jj+0]*tt[2] + w[4*jj+2]*tt[3];
            wz[3] = w[4*jj+1]*tt[2] + w[4*jj+3]*tt[3];*/
            D->value[4*jj+0] -= z[4*jj+0]*wz[0] + z[4*jj+2]*wz[1];
            D->value[4*jj+1] -= z[4*jj+1]*wz[0] + z[4*jj+3]*wz[1];
            D->value[4*jj+2] -= z[4*jj+0]*wz[2] + z[4*jj+2]*wz[3];
            D->value[4*jj+3] -= z[4*jj+1]*wz[2] + z[4*jj+3]*wz[3];

/*            D->value[4*jj+0] -= w[4*jj+0]*z[4*jj+0] + w[4*jj+2]*z[4*jj+1];
            D->value[4*jj+1] -= w[4*jj+1]*z[4*jj+0] + w[4*jj+3]*z[4*jj+1];
            D->value[4*jj+2] -= w[4*jj+0]*z[4*jj+2] + w[4*jj+2]*z[4*jj+3];
            D->value[4*jj+3] -= w[4*jj+1]*z[4*jj+2] + w[4*jj+3]*z[4*jj+3];*/
            break;
          case 3:
/*            wz[0] = tt[0]*z[9*jj+0] + tt[3]*z[9*jj+1] + tt[6]*z[9*jj+2];
            wz[1] = tt[1]*z[9*jj+0] + tt[4]*z[9*jj+1] + tt[7]*z[9*jj+2];
            wz[2] = tt[2]*z[9*jj+0] + tt[5]*z[9*jj+1] + tt[8]*z[9*jj+2];
            wz[3] = tt[0]*z[9*jj+3] + tt[3]*z[9*jj+4] + tt[6]*z[9*jj+5];
            wz[4] = tt[1]*z[9*jj+3] + tt[4]*z[9*jj+4] + tt[7]*z[9*jj+5];
            wz[5] = tt[2]*z[9*jj+3] + tt[5]*z[9*jj+4] + tt[8]*z[9*jj+5];
            wz[6] = tt[0]*z[9*jj+6] + tt[3]*z[9*jj+7] + tt[6]*z[9*jj+8];
            wz[7] = tt[1]*z[9*jj+6] + tt[4]*z[9*jj+7] + tt[7]*z[9*jj+8];
            wz[8] = tt[2]*z[9*jj+6] + tt[5]*z[9*jj+7] + tt[8]*z[9*jj+8];
*/
/*            wz[0] = z[9*jj+0];
            wz[1] = z[9*jj+1];
            wz[2] = z[9*jj+2];
            wz[3] = z[9*jj+3];
            wz[4] = z[9*jj+4];
            wz[5] = z[9*jj+5];
            wz[6] = z[9*jj+6];
            wz[7] = z[9*jj+7];
            wz[8] = z[9*jj+8];*/

            D->value[9*jj+0] -= w[9*jj+0]*wz[0] + w[9*jj+3]*wz[1] + w[9*jj+6]*wz[2];
            D->value[9*jj+1] -= w[9*jj+1]*wz[0] + w[9*jj+4]*wz[1] + w[9*jj+7]*wz[2];
            D->value[9*jj+2] -= w[9*jj+2]*wz[0] + w[9*jj+5]*wz[1] + w[9*jj+8]*wz[2];
            D->value[9*jj+3] -= w[9*jj+0]*wz[3] + w[9*jj+3]*wz[4] + w[9*jj+6]*wz[5];
            D->value[9*jj+4] -= w[9*jj+1]*wz[3] + w[9*jj+4]*wz[4] + w[9*jj+7]*wz[5];
            D->value[9*jj+5] -= w[9*jj+2]*wz[3] + w[9*jj+5]*wz[4] + w[9*jj+8]*wz[5];
            D->value[9*jj+6] -= w[9*jj+0]*wz[6] + w[9*jj+3]*wz[7] + w[9*jj+6]*wz[8];
            D->value[9*jj+7] -= w[9*jj+1]*wz[6] + w[9*jj+4]*wz[7] + w[9*jj+7]*wz[8];
            D->value[9*jj+8] -= w[9*jj+2]*wz[6] + w[9*jj+5]*wz[7] + w[9*jj+8]*wz[8];
            break;
          }
        }
      }
    }
    else
    {
      for(j=0;j<cw;j++)
      {
        jj = iw[j];
        if( zc[jj] )
        {
#if 1
          for(i=0;i<bnr;i++)
          {
            wz[i] = -w[bs*jj+i]*lu[0];
            for(ii=1;ii<bnr;ii++)
            {
              a = -w[bs*jj+ii*bnr+i];
              for(kk=0;kk<ii-1;kk++)
              {
                a -= wz[kk*bnr+i] * lu[ii*bnr+kk];
              }
              wz[ii*bnr+i] = a * lu[ii*bnr+ii];
            }
          }
          for(i=0;i<bnr;i++)
          {
            for(ii=bnr-1;ii>=0;ii--)
            {
              a = wz[ii*bnr+i];
              for(kk=ii+1;kk<bnr;kk++)
              {
                a -= wz[kk*bnr+i] * lu[ii*bnr+kk];
              }
              wz[ii*bnr+i] = a;
            }
          }
#endif
          switch(bnr)
          {
          case 1:
/*            D->value[jj] -= z[jj]*w[jj];
            D->value[jj] -= z[jj]*w[jj]*tt[0];*/
            D->value[jj] -= z[jj]*wz[0];
            break;
          case 2:
/*            wz[0] = w[4*jj+0]*tt[0] + w[4*jj+2]*tt[1];
            wz[1] = w[4*jj+1]*tt[0] + w[4*jj+3]*tt[1];
            wz[2] = w[4*jj+0]*tt[2] + w[4*jj+2]*tt[3];
            wz[3] = w[4*jj+1]*tt[2] + w[4*jj+3]*tt[3];
            */
            D->value[4*jj+0] -= z[4*jj+0]*wz[0] + z[4*jj+2]*wz[1];
            D->value[4*jj+1] -= z[4*jj+1]*wz[0] + z[4*jj+3]*wz[1];
            D->value[4*jj+2] -= z[4*jj+0]*wz[2] + z[4*jj+2]*wz[3];
            D->value[4*jj+3] -= z[4*jj+1]*wz[2] + z[4*jj+3]*wz[3];

/*            D->value[4*jj+0] -= w[4*jj+0]*z[4*jj+0] + w[4*jj+2]*z[4*jj+1];
            D->value[4*jj+1] -= w[4*jj+1]*z[4*jj+0] + w[4*jj+3]*z[4*jj+1];
            D->value[4*jj+2] -= w[4*jj+0]*z[4*jj+2] + w[4*jj+2]*z[4*jj+3];
            D->value[4*jj+3] -= w[4*jj+1]*z[4*jj+2] + w[4*jj+3]*z[4*jj+3];*/
            break;
          case 3:
/*            wz[0] = tt[0]*z[9*jj+0] + tt[3]*z[9*jj+1] + tt[6]*z[9*jj+2];
            wz[1] = tt[1]*z[9*jj+0] + tt[4]*z[9*jj+1] + tt[7]*z[9*jj+2];
            wz[2] = tt[2]*z[9*jj+0] + tt[5]*z[9*jj+1] + tt[8]*z[9*jj+2];
            wz[3] = tt[0]*z[9*jj+3] + tt[3]*z[9*jj+4] + tt[6]*z[9*jj+5];
            wz[4] = tt[1]*z[9*jj+3] + tt[4]*z[9*jj+4] + tt[7]*z[9*jj+5];
            wz[5] = tt[2]*z[9*jj+3] + tt[5]*z[9*jj+4] + tt[8]*z[9*jj+5];
            wz[6] = tt[0]*z[9*jj+6] + tt[3]*z[9*jj+7] + tt[6]*z[9*jj+8];
            wz[7] = tt[1]*z[9*jj+6] + tt[4]*z[9*jj+7] + tt[7]*z[9*jj+8];
            wz[8] = tt[2]*z[9*jj+6] + tt[5]*z[9*jj+7] + tt[8]*z[9*jj+8];
*/
/*            wz[0] = z[9*jj+0];
            wz[1] = z[9*jj+1];
            wz[2] = z[9*jj+2];
            wz[3] = z[9*jj+3];
            wz[4] = z[9*jj+4];
            wz[5] = z[9*jj+5];
            wz[6] = z[9*jj+6];
            wz[7] = z[9*jj+7];
            wz[8] = z[9*jj+8];*/

            D->value[9*jj+0] -= w[9*jj+0]*wz[0] + w[9*jj+3]*wz[1] + w[9*jj+6]*wz[2];
            D->value[9*jj+1] -= w[9*jj+1]*wz[0] + w[9*jj+4]*wz[1] + w[9*jj+7]*wz[2];
            D->value[9*jj+2] -= w[9*jj+2]*wz[0] + w[9*jj+5]*wz[1] + w[9*jj+8]*wz[2];
            D->value[9*jj+3] -= w[9*jj+0]*wz[3] + w[9*jj+3]*wz[4] + w[9*jj+6]*wz[5];
            D->value[9*jj+4] -= w[9*jj+1]*wz[3] + w[9*jj+4]*wz[4] + w[9*jj+7]*wz[5];
            D->value[9*jj+5] -= w[9*jj+2]*wz[3] + w[9*jj+5]*wz[4] + w[9*jj+8]*wz[5];
            D->value[9*jj+6] -= w[9*jj+0]*wz[6] + w[9*jj+3]*wz[7] + w[9*jj+6]*wz[8];
            D->value[9*jj+7] -= w[9*jj+1]*wz[6] + w[9*jj+4]*wz[7] + w[9*jj+7]*wz[8];
            D->value[9*jj+8] -= w[9*jj+2]*wz[6] + w[9*jj+5]*wz[7] + w[9*jj+8]*wz[8];
            break;
          }
        }
      }
    }
#endif
    /* drop U */
    nnz = 0;
    for(j=0;j<cz;j++)
    {
      jj = iz[j];
      switch(bnr)
      {
      case 1:
        t       = fabs(z[jj]);
        break;
      case 2:
        t       = fabs(z[4*jj+0]) + fabs(z[4*jj+1]);
        t      += fabs(z[4*jj+2]) + fabs(z[4*jj+3]);
/*        t       = sqrt(t);*/
        break;
      case 3:
        t       = fabs(z[9*jj+0]) + fabs(z[9*jj+1]) + fabs(z[9*jj+2]);
        t      += fabs(z[9*jj+3]) + fabs(z[9*jj+4]) + fabs(z[9*jj+5]);
        t      += fabs(z[9*jj+6]) + fabs(z[9*jj+7]) + fabs(z[9*jj+8]);
/*        t       = sqrt(t);*/
        break;
      }
      if( t>toldd )
      {
        iz[nnz++]    = jj;
        tmp[jj] = t;
      }
      else
      {
        switch(bnr)
        {
        case 1:
          z[k] = z[k] + fabs(z[jj]);
          break;
        case 2:
          z[4*k+0] += fabs(z[4*jj+0]);
          z[4*k+1] += fabs(z[4*jj+1]);
          z[4*k+1] += fabs(z[4*jj+2]);
          z[4*k+1] += fabs(z[4*jj+3]);
          break;
        case 3:
          z[9*k+0] += fabs(z[9*jj+0]);
          z[9*k+1] += fabs(z[9*jj+1]);
          z[9*k+2] += fabs(z[9*jj+2]);
          z[9*k+3] += fabs(z[9*jj+3]);
          z[9*k+4] += fabs(z[9*jj+4]);
          z[9*k+5] += fabs(z[9*jj+5]);
          z[9*k+6] += fabs(z[9*jj+6]);
          z[9*k+7] += fabs(z[9*jj+7]);
          z[9*k+8] += fabs(z[9*jj+8]);
          break;
        }
        zc[jj] = 0;
        tmp[jj] = 0.0;
      }
    }
    len = _min(lfil,nnz);
    lis_sort_di(0,nnz-1,tmp,iz);
    lis_sort_i(0,len-1,iz);
    U->nnz[k] = len;
    if( len>0 )
    {
      U->index[k] = (LIS_INT *)malloc(len*sizeof(LIS_INT));
      U->value[k] = (LIS_SCALAR *)malloc(bs*len*sizeof(LIS_SCALAR));
    }
    for(j=0;j<len;j++)
    {
      jj = iz[j];
      U->index[k][j] = jj;
/*      U->value[k][j] = z[jj];*/
      memcpy(&U->value[k][bs*j],&z[bs*jj],bs*sizeof(LIS_SCALAR));
/*      for(i=0;i<bnr;i++)
      {
        printf("U:k=%d j=%d",k,jj);
        for(ii=0;ii<bnr;ii++)
        {
          printf(" %e",z[bs*jj+ii*bnr+i]);
        }
        printf("\n");
      }*/
    }
    for(j=len;j<cz;j++) zc[iz[j]] = 0;
    cz = nnz;
/*    printf("k=%d U:lfil=%d nnz=%d len=%d\n",k,lfil,nnz,len);*/

    /* drop L */
    nnz = 0;
    for(j=0;j<cw;j++)
    {
      jj = iw[j];
      switch(bnr)
      {
      case 1:
        t       = fabs(w[jj]);
        break;
      case 2:
        t       = fabs(w[4*jj+0]) + fabs(w[4*jj+1]);
        t      += fabs(w[4*jj+2]) + fabs(w[4*jj+3]);
/*        t       = sqrt(t);*/
        break;
      case 3:
        t       = fabs(w[9*jj+0]) + fabs(w[9*jj+1]) + fabs(w[9*jj+2]);
        t      += fabs(w[9*jj+3]) + fabs(w[9*jj+4]) + fabs(w[9*jj+5]);
        t      += fabs(w[9*jj+6]) + fabs(w[9*jj+7]) + fabs(w[9*jj+8]);
/*        t       = sqrt(t);*/
        break;
      }
/*      printf("k=%d L:jj=%d t=%e tol=%e\n",k,jj,t,toldd);*/
      if( t>toldd )
      {
        iw[nnz++]    = jj;
        tmp[jj] = t;
      }
      else
      {
        wc[jj] = 0;
        tmp[jj] = 0.0;
      }
    }
    len = _min(lfil,nnz);
    lis_sort_di(0,nnz-1,tmp,iw);
    lis_sort_i(0,len-1,iw);
    L->nnz[k] = len;
    if( len>0 )
    {
      L->index[k] = (LIS_INT *)malloc(len*sizeof(LIS_INT));
      L->value[k] = (LIS_SCALAR *)malloc(bs*len*sizeof(LIS_SCALAR));
    }
    for(j=0;j<len;j++)
    {
      jj = iw[j];
      L->index[k][j] = jj;

#if 1
      for(i=0;i<bnr;i++)
      {
        L->value[k][bs*j+i] = -w[bs*jj+i]*lu[0];
        for(ii=1;ii<bnr;ii++)
        {
          a = -w[bs*jj+ii*bnr+i];
          for(kk=0;kk<ii-1;kk++)
          {
            a -= L->value[k][bs*j+kk*bnr+i] * lu[ii*bnr+kk];
          }
          L->value[k][bs*j+ii*bnr+i] = a * lu[ii*bnr+ii];
        }
      }
      for(i=0;i<bnr;i++)
      {
        for(ii=bnr-1;ii>=0;ii--)
        {
          a = L->value[k][bs*j+ii*bnr+i];
          for(kk=ii+1;kk<bnr;kk++)
          {
            a -= L->value[k][bs*j+kk*bnr+i] * lu[ii*bnr+kk];
          }
          L->value[k][bs*j+ii*bnr+i] = a;
        }
      }
#endif
#if 0
      switch(bnr)
      {
      case 1:
        L->value[k][j] = w[jj]*tt[0];
        break;
      case 2:
        L->value[k][4*j+0] = w[4*jj+0]*tt[0] + w[4*jj+2]*tt[1];
        L->value[k][4*j+1] = w[4*jj+1]*tt[0] + w[4*jj+3]*tt[1];
        L->value[k][4*j+2] = w[4*jj+0]*tt[2] + w[4*jj+2]*tt[3];
        L->value[k][4*j+3] = w[4*jj+1]*tt[2] + w[4*jj+3]*tt[3];
        break;
      case 3:
        L->value[k][9*j+0] = w[9*jj+0]*tt[0] + w[9*jj+3]*tt[1] + w[9*jj+6]*tt[2];
        L->value[k][9*j+1] = w[9*jj+1]*tt[0] + w[9*jj+4]*tt[1] + w[9*jj+7]*tt[2];
        L->value[k][9*j+2] = w[9*jj+2]*tt[0] + w[9*jj+5]*tt[1] + w[9*jj+8]*tt[2];
        L->value[k][9*j+3] = w[9*jj+0]*tt[3] + w[9*jj+3]*tt[4] + w[9*jj+6]*tt[5];
        L->value[k][9*j+4] = w[9*jj+1]*tt[3] + w[9*jj+4]*tt[4] + w[9*jj+7]*tt[5];
        L->value[k][9*j+5] = w[9*jj+2]*tt[3] + w[9*jj+5]*tt[4] + w[9*jj+8]*tt[5];
        L->value[k][9*j+6] = w[9*jj+0]*tt[6] + w[9*jj+3]*tt[7] + w[9*jj+6]*tt[8];
        L->value[k][9*j+7] = w[9*jj+1]*tt[6] + w[9*jj+4]*tt[7] + w[9*jj+7]*tt[8];
        L->value[k][9*j+8] = w[9*jj+2]*tt[6] + w[9*jj+5]*tt[7] + w[9*jj+8]*tt[8];
        break;
      }
/*      for(i=0;i<bnr;i++)
      {
        printf("L:k=%d j=%d",k,jj);
        for(ii=0;ii<bnr;ii++)
        {
          printf(" %e",L->value[k][bs*j+ii*bnr+i]);
        }
        printf("\n");
      }*/
#endif
    }
    for(j=len;j<cw;j++) wc[iw[j]] = 0;
    cw = nnz;
/*    printf("k=%d L:lfil=%d nnz=%d len=%d\n",k,lfil,nnz,len);*/


    /**/
    for(j=0;j<cz;j++) zc[iz[j]] = 0;
    for(j=0;j<cw;j++) wc[iw[j]] = 0;
    if(U->nnz[k]>0)
    {
      jj     = iz[0];
      zc[k]  = 0;
      zl[k]  = zl[jj];
      zl[jj] = k;
    }
    if(L->nnz[k]>0)
    {
      jj     = iw[0];
      wc[k]  = 0;
      wl[k]  = wl[jj];
      wl[jj] = k;
    }
  }

  precon->L  = L;
  precon->U  = U;
  precon->WD  = D;

  lis_free2(12,tmp,w,z,iw,wc,wl,iz,zc,zl,ptr,index,value);

  LIS_DEBUG_FUNC_OUT;
  return LIS_SUCCESS;
}



#undef __FUNC__
#define __FUNC__ "lis_psolve_iluc"
LIS_INT lis_psolve_iluc(LIS_SOLVER solver, LIS_VECTOR B, LIS_VECTOR X)
{
#ifdef _OPENMP
  LIS_INT i,j,jj,n;
  LIS_INT nprocs,my_rank,is,ie;
  LIS_SCALAR w;
  LIS_SCALAR *b,*x;
  LIS_MATRIX_ILU L,U;
  LIS_VECTOR D;
  LIS_PRECON  precon;
  LIS_QUAD_DECLAR;
  #ifdef USE_QUAD_PRECISION
    LIS_QUAD w1,w2;
    LIS_SCALAR *xl;
  #endif


  LIS_DEBUG_FUNC_IN;

  precon = solver->precon;
  L = precon->L;
  U = precon->U;
  D = precon->D;
  n = L->n;
  b = B->value;
  x = X->value;
  #ifdef USE_QUAD_PRECISION
    xl = X->value_lo;
  #endif

  #ifdef USE_QUAD_PRECISION
    if( B->precision==LIS_PRECISION_DEFAULT )
    {
  #endif
      lis_vector_copy(B,X);
      nprocs = omp_get_max_threads();
      #pragma omp parallel private(i,j,jj,is,ie,my_rank,w)
      {
        my_rank = omp_get_thread_num();
        LIS_GET_ISIE(my_rank,nprocs,n,is,ie);

        for(i=is; i<ie; i++)
        {
          for(j=0;j<L->nnz[i];j++)
          {
            jj     = L->index[i][j];
            x[jj] -= L->value[i][j] * x[i];
          }
        }
        for(i=ie-1; i>=is; i--)
        {
          w = x[i];
          for(j=0;j<U->nnz[i];j++)
          {
            jj = U->index[i][j];
            w -= U->value[i][j] * x[jj];
          }
          x[i] = w * D->value[i];
        }
      }
  #ifdef USE_QUAD_PRECISION
    }
    else
    {
      lis_vector_copyex_mm(B,X);
      nprocs = omp_get_max_threads();
      #ifndef USE_SSE2
        #pragma omp parallel private(i,j,jj,is,ie,my_rank,w1,p1,p2,tq,bhi,blo,chi,clo,sh,sl,th,tl,eh,el)
      #else
        #pragma omp parallel private(i,j,jj,is,ie,my_rank,w1,bh,ch,sh,wh,th,bl,cl,sl,wl,tl,p1,p2,t0,t1,t2,eh)
      #endif
      {
        my_rank = omp_get_thread_num();
        LIS_GET_ISIE(my_rank,nprocs,n,is,ie);
        for(i=is; i<ie; i++)
        {
          for(j=0;j<L->nnz[i];j++)
          {
            jj     = L->index[i][j];
/*            if( jj>=is && jj<ie )*/
            {
              #ifndef USE_SSE2
                LIS_QUAD_FMAD(x[jj],xl[jj],x[jj],xl[jj],x[i],xl[i],-L->value[i][j]);
              #else
                LIS_QUAD_FMAD_SSE2(x[jj],xl[jj],x[jj],xl[jj],x[i],xl[i],-L->value[i][j]);
              #endif
            }
          }
        }
        for(i=ie-1; i>=is; i--)
        {
          w1.hi = x[i];
          w1.lo = xl[i];
          for(j=0;j<U->nnz[i];j++)
          {
            jj = U->index[i][j];
            #ifndef USE_SSE2
              LIS_QUAD_FMAD(w1.hi,w1.lo,w1.hi,w1.lo,x[jj],xl[jj],-U->value[i][j]);
            #else
              LIS_QUAD_FMAD_SSE2(w1.hi,w1.lo,w1.hi,w1.lo,x[jj],xl[jj],-U->value[i][j]);
            #endif
          }
          #ifndef USE_SSE2
            LIS_QUAD_MULD(x[i],xl[i],w1.hi,w1.lo,D->value[i]);
          #else
            LIS_QUAD_MULD_SSE2(x[i],xl[i],w1.hi,w1.lo,D->value[i]);
          #endif
        }
      }
    }
  #endif

  LIS_DEBUG_FUNC_OUT;
  return LIS_SUCCESS;

#else
  LIS_INT i,j,jj,n;
  LIS_SCALAR w;
  LIS_SCALAR *b,*x;
  LIS_MATRIX_ILU L,U;
  LIS_VECTOR D;
  LIS_PRECON  precon;
  LIS_QUAD_DECLAR;
  #ifdef USE_QUAD_PRECISION
    LIS_QUAD w1,w2;
    LIS_SCALAR *xl;
  #endif

  /*
   *  LUx = b
   *  LU  = (D + L*A) * (I + D^-1 * U*A)
   */

  LIS_DEBUG_FUNC_IN;

  precon = solver->precon;
  L = precon->L;
  U = precon->U;
  D = precon->D;
  n = L->n;
  b = B->value;
  x = X->value;
  #ifdef USE_QUAD_PRECISION
    xl = X->value_lo;
  #endif

  #ifdef USE_QUAD_PRECISION
    if( B->precision==LIS_PRECISION_DEFAULT )
    {
  #endif
      lis_vector_copy(B,X);
      for(i=0; i<n; i++)
      {
        for(j=0;j<L->nnz[i];j++)
        {
          jj     = L->index[i][j];
          x[jj] -= L->value[i][j] * x[i];
        }
/*        x[i] = x[i] * L->work[i];*/
      }
      for(i=n-1; i>=0; i--)
      {
        w = x[i];
        for(j=0;j<U->nnz[i];j++)
        {
          jj = U->index[i][j];
          w -= U->value[i][j] * x[jj];
        }
        x[i] = w * D->value[i];
      }
  #ifdef USE_QUAD_PRECISION
    }
    else
    {
      lis_vector_copyex_mm(B,X);
      for(i=0; i<n; i++)
      {
        for(j=0;j<L->nnz[i];j++)
        {
          jj     = L->index[i][j];
          #ifndef USE_SSE2
            LIS_QUAD_FMAD(x[jj],xl[jj],x[jj],xl[jj],x[i],xl[i],-L->value[i][j]);
          #else
            LIS_QUAD_FMAD_SSE2(x[jj],xl[jj],x[jj],xl[jj],x[i],xl[i],-L->value[i][j]);
          #endif
          /* x[jj] -= L->value[i][j] * x[i]; */
        }
      }
      for(i=n-1; i>=0; i--)
      {
        w1.hi = x[i];
        w1.lo = xl[i];
        /* t = x[i]; */
        for(j=0;j<U->nnz[i];j++)
        {
          jj = U->index[i][j];
          #ifndef USE_SSE2
            LIS_QUAD_FMAD(w1.hi,w1.lo,w1.hi,w1.lo,x[jj],xl[jj],-U->value[i][j]);
          #else
            LIS_QUAD_FMAD_SSE2(w1.hi,w1.lo,w1.hi,w1.lo,x[jj],xl[jj],-U->value[i][j]);
          #endif
          /* t -= U->value[i][j] * x[jj]; */
        }
        #ifndef USE_SSE2
          LIS_QUAD_MULD(x[i],xl[i],w1.hi,w1.lo,D->value[i]);
        #else
          LIS_QUAD_MULD_SSE2(x[i],xl[i],w1.hi,w1.lo,D->value[i]);
        #endif
        /* x[i] = t * D->value[i]; */
      }
    }
  #endif

  LIS_DEBUG_FUNC_OUT;
  return LIS_SUCCESS;

#endif
}

#undef __FUNC__
#define __FUNC__ "lis_psolve_iluc_bsr"
LIS_INT lis_psolve_iluc_bsr(LIS_SOLVER solver, LIS_VECTOR B, LIS_VECTOR X)
{
  LIS_INT i,j,jj,nr,bnr,ii,bs;
  LIS_SCALAR w[9],t;
  LIS_SCALAR *b,*x;
  LIS_MATRIX_ILU L,U;
  LIS_MATRIX_DIAG D;
  LIS_PRECON  precon;

  /*
   *  LUx = b
   *  LU  = (D + L*A) * (I + D^-1 * U*A)
   */

  LIS_DEBUG_FUNC_IN;

  precon = solver->precon;
  L = precon->L;
  U = precon->U;
  D = precon->WD;
  b = B->value;
  x = X->value;
  nr = solver->A->nr;
  bnr = solver->A->bnr;
  bs  = bnr*bnr;

  lis_vector_copy(B,X);
  for(i=0; i<nr; i++)
  {
    for(j=0;j<L->nnz[i];j++)
    {
      jj     = L->index[i][j];
/*      memcpy(w,&x[bnr*i],bnr*sizeof(LIS_SCALAR));*/
      switch(bnr)
      {
      case 1:
        x[jj] -= L->value[i][j] * x[i];
        break;
      case 2:
        x[2*jj+0] -= L->value[i][4*j]   * x[2*i+0];
        x[2*jj+0] -= L->value[i][4*j+2] * x[2*i+1];
        x[2*jj+1] -= L->value[i][4*j+1] * x[2*i+0];
        x[2*jj+1] -= L->value[i][4*j+3] * x[2*i+1];
        break;
      case 3:
        x[3*jj]   -= L->value[i][9*j]   * x[3*i+0] + L->value[i][9*j+3] * x[3*i+1] + L->value[i][9*j+6] * x[3*i+2];
        x[3*jj+1] -= L->value[i][9*j+1] * x[3*i+0] + L->value[i][9*j+4] * x[3*i+1] + L->value[i][9*j+7] * x[3*i+2];
        x[3*jj+2] -= L->value[i][9*j+2] * x[3*i+0] + L->value[i][9*j+5] * x[3*i+1] + L->value[i][9*j+8] * x[3*i+2];
        break;
      }
    }
  }
  for(i=nr-1; i>=0; i--)
  {
/*    w = x[i];*/
    memcpy(w,&x[bnr*i],bnr*sizeof(LIS_SCALAR));
    for(j=0;j<U->nnz[i];j++)
    {
      jj = U->index[i][j];
      switch(bnr)
      {
      case 1:
        w[0] -= U->value[i][j] * x[jj];
        break;
      case 2:
        w[0] -= U->value[i][4*j]   * x[2*jj];
        w[0] -= U->value[i][4*j+2] * x[2*jj+1];
        w[1] -= U->value[i][4*j+1] * x[2*jj];
        w[1] -= U->value[i][4*j+3] * x[2*jj+1];
        break;
      case 3:
        w[0] -= U->value[i][9*j]   * x[3*jj] + U->value[i][9*j+3] * x[3*jj+1] + U->value[i][9*j+6] * x[3*jj+2];
        w[1] -= U->value[i][9*j+1] * x[3*jj] + U->value[i][9*j+4] * x[3*jj+1] + U->value[i][9*j+7] * x[3*jj+2];
        w[2] -= U->value[i][9*j+2] * x[3*jj] + U->value[i][9*j+5] * x[3*jj+1] + U->value[i][9*j+8] * x[3*jj+2];
        break;
      }
    }
#if 0
    switch(bnr)
    {
    case 1:
      x[i] = w[0] * D->value[i];
      break;
    case 2:
      x[2*i+0] = D->value[4*i]   * w[0] + D->value[4*i+2] * w[1];
      x[2*i+1] = D->value[4*i+1] * w[0] + D->value[4*i+3] * w[1];
      break;
    case 3:
      x[3*i+0] = D->value[9*i]   * w[0] + D->value[9*i+3] * w[1] + D->value[9*i+6] * w[2];
      x[3*i+1] = D->value[9*i+1] * w[0] + D->value[9*i+4] * w[1] + D->value[9*i+7] * w[2];
      x[3*i+2] = D->value[9*i+2] * w[0] + D->value[9*i+5] * w[1] + D->value[9*i+8] * w[2];
      break;
    }
#else
    for(ii=0;ii<bnr;ii++)
    {
      t = w[ii];
      for(jj=0;jj<ii;jj++)
      {
        t -= D->value[bs*i+jj*bnr+ii] * x[bnr*i+jj];
      }
      x[bnr*i+ii] = t;
    }
    for(ii=bnr-1;ii>=0;ii--)
    {
      t = x[bnr*i+ii];
      for(jj=ii+1;jj<bnr;jj++)
      {
        t -= D->value[bs*i+jj*bnr+ii] * x[bnr*i+jj];
      }
      x[bnr*i+ii] = t * D->value[bs*i+ii*bnr+ii];
    }
#endif
  }

  LIS_DEBUG_FUNC_OUT;
  return LIS_SUCCESS;
}

#undef __FUNC__
#define __FUNC__ "lis_psolvet_iluc"
LIS_INT lis_psolvet_iluc(LIS_SOLVER solver, LIS_VECTOR B, LIS_VECTOR X)
{
#ifdef _OPENMP
  LIS_INT i,j,jj,n,is,ie,my_rank,nprocs;
  LIS_SCALAR w;
  LIS_SCALAR *b,*x;
  LIS_MATRIX_ILU L,U;
  LIS_VECTOR D;
  LIS_PRECON  precon;
  LIS_QUAD_DECLAR;
  #ifdef USE_QUAD_PRECISION
    LIS_QUAD w1,w2;
    LIS_SCALAR *xl;
  #endif

  /*
   *  (LU)'x = b
   *  U'x=b
   *  L'x=b
   */


  LIS_DEBUG_FUNC_IN;

  precon = solver->precon;
  L = precon->L;
  U = precon->U;
  D = precon->D;
  n = L->n;
  b = B->value;
  x = X->value;
  #ifdef USE_QUAD_PRECISION
    xl = X->value_lo;
  #endif

  #ifdef USE_QUAD_PRECISION
    if( B->precision==LIS_PRECISION_DEFAULT )
    {
  #endif
      lis_vector_copy(B,X);
      nprocs = omp_get_max_threads();
      #pragma omp parallel private(i,j,jj,is,ie,my_rank,w)
      {
        my_rank = omp_get_thread_num();
        LIS_GET_ISIE(my_rank,nprocs,n,is,ie);
        for(i=is; i<ie; i++)
        {
          x[i] = x[i] * D->value[i];
          for(j=0;j<U->nnz[i];j++)
          {
            jj     = U->index[i][j];
            x[jj] -= U->value[i][j] * x[i];
          }
        }
        for(i=ie-2; i>=is; i--)
        {
          w = x[i];
          for(j=0;j<L->nnz[i];j++)
          {
            jj  = L->index[i][j];
            w  -= L->value[i][j] * x[jj];
          }
          x[i] = w;
        }
      }
  #ifdef USE_QUAD_PRECISION
    }
    else
    {
      lis_vector_copyex_mm(B,X);
      nprocs = omp_get_max_threads();
      #ifndef USE_SSE2
        #pragma omp parallel private(i,j,jj,is,ie,my_rank,w1,p1,p2,tq,bhi,blo,chi,clo,sh,sl,th,tl,eh,el)
      #else
        #pragma omp parallel private(i,j,jj,is,ie,my_rank,w1,bh,ch,sh,wh,th,bl,cl,sl,wl,tl,p1,p2,t0,t1,t2,eh)
      #endif
      {
        my_rank = omp_get_thread_num();
        LIS_GET_ISIE(my_rank,nprocs,n,is,ie);
        for(i=is; i<ie; i++)
        {
          #ifndef USE_SSE2
            LIS_QUAD_MULD(x[i],xl[i],x[i],xl[i],D->value[i]);
          #else
            LIS_QUAD_MULD_SSE2(x[i],xl[i],x[i],xl[i],D->value[i]);
          #endif
          /* x[i] = x[i] * D->value[i]; */
          for(j=0;j<U->nnz[i];j++)
          {
            jj     = U->index[i][j];
            #ifndef USE_SSE2
              LIS_QUAD_FMAD(x[jj],xl[jj],x[jj],xl[jj],x[i],xl[i],-U->value[i][j]);
            #else
              LIS_QUAD_FMAD_SSE2(x[jj],xl[jj],x[jj],xl[jj],x[i],xl[i],-U->value[i][j]);
            #endif
            /* x[jj] -= U->value[i][j] * x[i]; */
          }
        }
        for(i=ie-2; i>=is; i--)
        {
          w1.hi = x[i];
          w1.lo = xl[i];
          /* t = x[i]; */
          for(j=0;j<L->nnz[i];j++)
          {
            jj  = L->index[i][j];
            #ifndef USE_SSE2
              LIS_QUAD_FMAD(w1.hi,w1.lo,w1.hi,w1.lo,x[jj],xl[jj],-L->value[i][j]);
            #else
              LIS_QUAD_FMAD_SSE2(w1.hi,w1.lo,w1.hi,w1.lo,x[jj],xl[jj],-L->value[i][j]);
            #endif
            /* t  -= L->value[i][j] * x[jj]; */
          }
          x[i]  = w1.hi;
          xl[i] = w1.lo;
          /* x[i] = t; */
        }
      }
    }
  #endif

  LIS_DEBUG_FUNC_OUT;
  return LIS_SUCCESS;
#else
  LIS_INT i,j,jj,n;
  LIS_SCALAR w;
  LIS_SCALAR *b,*x;
  LIS_MATRIX_ILU L,U;
  LIS_VECTOR D;
  LIS_PRECON  precon;
  LIS_QUAD_DECLAR;
  #ifdef USE_QUAD_PRECISION
    LIS_QUAD w1,w2;
    LIS_SCALAR *xl;
  #endif

  /*
   *  (LU)'x = b
   *  U'x=b
   *  L'x=b
   */


  LIS_DEBUG_FUNC_IN;

  precon = solver->precon;
  L = precon->L;
  U = precon->U;
  D = precon->D;
  n = L->n;
  b = B->value;
  x = X->value;
  #ifdef USE_QUAD_PRECISION
    xl = X->value_lo;
  #endif

  #ifdef USE_QUAD_PRECISION
    if( B->precision==LIS_PRECISION_DEFAULT )
    {
  #endif
      lis_vector_copy(B,X);
      for(i=0; i<n; i++)
      {
        x[i] = x[i] * D->value[i];
        for(j=0;j<U->nnz[i];j++)
        {
          jj     = U->index[i][j];
          x[jj] -= U->value[i][j] * x[i];
        }
      }
      for(i=n-2; i>=0; i--)
      {
        w = x[i];
        for(j=0;j<L->nnz[i];j++)
        {
          jj  = L->index[i][j];
          w  -= L->value[i][j] * x[jj];
        }
        x[i] = w;
      }
  #ifdef USE_QUAD_PRECISION
    }
    else
    {
      lis_vector_copyex_mm(B,X);
      for(i=0; i<n; i++)
      {
        #ifndef USE_SSE2
          LIS_QUAD_MULD(x[i],xl[i],x[i],xl[i],D->value[i]);
        #else
          LIS_QUAD_MULD_SSE2(x[i],xl[i],x[i],xl[i],D->value[i]);
        #endif
        /* x[i] = x[i] * D->value[i]; */
        for(j=0;j<U->nnz[i];j++)
        {
          jj     = U->index[i][j];
          #ifndef USE_SSE2
            LIS_QUAD_FMAD(x[jj],xl[jj],x[jj],xl[jj],x[i],xl[i],-U->value[i][j]);
          #else
            LIS_QUAD_FMAD_SSE2(x[jj],xl[jj],x[jj],xl[jj],x[i],xl[i],-U->value[i][j]);
          #endif
          /* x[jj] -= U->value[i][j] * x[i]; */
        }
      }
      for(i=n-2; i>=0; i--)
      {
        w1.hi = x[i];
        w1.lo = xl[i];
        /* t = x[i]; */
        for(j=0;j<L->nnz[i];j++)
        {
          jj  = L->index[i][j];
          #ifndef USE_SSE2
            LIS_QUAD_FMAD(w1.hi,w1.lo,w1.hi,w1.lo,x[jj],xl[jj],-L->value[i][j]);
          #else
            LIS_QUAD_FMAD_SSE2(w1.hi,w1.lo,w1.hi,w1.lo,x[jj],xl[jj],-L->value[i][j]);
          #endif
          /* t  -= L->value[i][j] * x[jj]; */
        }
        x[i]  = w1.hi;
        xl[i] = w1.lo;
        /* x[i] = t; */
      }
    }
  #endif

  LIS_DEBUG_FUNC_OUT;
  return LIS_SUCCESS;
#endif
}

