GSI3.3版本，运行官网单点观测case，出现SIGSEGV, segmentation fault occurred

哞哞三年半

一共四步，请大神帮我看看出错原因@大马哈 @Уēｓ、尜玉
1.GSIV3.3版本编译信息：ifort/icc Version 13.0.1, mpirun Version 4.1.0 编译成功，正常生成gsi.exe；

2.运行官网的单点观测实验，run_gsi.ksh脚本如下：
#!/bin/ksh
#####################################################
# machine set up (users should change this part)
#####################################################
#
# LSF batch script to run an MPI application


#
#run the executable
#####################################################
# machine set up (users should change this part)
#####################################################


#
#run the executable
set -x


#
# GSIPROC = processor number used for GSI analysis
#------------------------------------------------
  GSIPROC=6
  ARCH='LINUX'
# Supported configurations:
            # IBM_LSF,
            # LINUX, LINUX_LSF, LINUX_PBS,
            # DARWIN_PGI
#
#####################################################
# case set up (users should change this part)
#####################################################
#
# ANAL_TIME= analysis time  (YYYYMMDDHH)
# WORK_ROOT= working directory, where GSI runs
# PREPBURF = path of PreBUFR conventional obs
# BK_FILE  = path and name of background file
# OBS_ROOT = path of observations files
# FIX_ROOT = path of fix files
# GSI_EXE  = path and name of the gsi executable
  ANAL_TIME=2011032212
  WORK_ROOT=/SWNEW/TEST/comGSI_v3.3/run/testrun
  OBS_ROOT=/SWNEW/TEST/comGSI_v3.3/run/testdata/2011032212/obs
  PREPBUFR=${OBS_ROOT}/gdas1.t12z.prepbufr.nr
  BK_FILE=/SWNEW/TEST/comGSI_v3.3/run/testdata/2011032212/wrfinput_d01_ARW_2011-03-22_12
  FIX_ROOT=/SWNEW/TEST/comGSI_v3.3/fix
# CRTM_ROOT=/SWNEW/TEST/CRTM_REL-2.1.3_Little_Endian
  CRTM_ROOT=/SWNEW/TEST/CRTM_REL-2.1.3
  GSI_EXE=/SWNEW/TEST/comGSI_v3.3/run/gsi.exe


#------------------------------------------------
# bk_core= which WRF core is used as background (NMM or ARW)
# bkcv_option= which background error covariance and parameter will be used
#              (GLOBAL or NAM)
# if_clean = clean  : delete temperal files in working directory (default)
#            no     : leave running directory as is (this is for debug only)
  bk_core=ARW
  bkcv_option=GLOBAL
  if_clean=no
#
#
#####################################################
# Users should NOT change script after this point
#####################################################
#
BYTE_ORDER=Big_Endian
#BYTE_ORDER=Little_Endian


case $ARCH in
   'IBM_LSF')
      ###### IBM LSF (Load Sharing Facility)
      RUN_COMMAND="mpirun.lsf " ;;


   'LINUX')
      if [ $GSIPROC = 1 ]; then
         #### Linux workstation - single processor
         RUN_COMMAND=""
      else
         ###### Linux workstation -  mpi run
       #RUN_COMMAND="mpirun -np ${GSIPROC} -machinefile ./hosts"
       RUN_COMMAND="mpirun -machinefile /SWNEW/TEST/comGSI_v3.3/run/hosts -np ${GSIPROC}"
      fi ;;


   'LINUX_LSF')
      ###### LINUX LSF (Load Sharing Facility)
      RUN_COMMAND="mpirun.lsf " ;;


   'LINUX_PBS')
      #### Linux cluster PBS (Portable Batch System)
      RUN_COMMAND="mpirun -np ${GSIPROC} " ;;


   'DARWIN_PGI')
      ### Mac - mpi run
      if [ $GSIPROC = 1 ]; then
         #### Mac workstation - single processor
         RUN_COMMAND=""
      else
         ###### Mac workstation -  mpi run
         RUN_COMMAND="mpirun -np ${GSIPROC} -machinefile ~/mach "
      fi ;;


   * )
     print "error: $ARCH is not a supported platform configuration."
     exit 1 ;;
esac




##################################################################################
# Check GSI needed environment variables are defined and exist
#

# Make sure ANAL_TIME is defined and in the correct format
if [ ! "${ANAL_TIME}" ]; then
  echo "ERROR: \$ANAL_TIME is not defined!"
  exit 1
fi


# Make sure WORK_ROOT is defined and exists
if [ ! "${WORK_ROOT}" ]; then
  echo "ERROR: \$WORK_ROOT is not defined!"
  exit 1
fi


# Make sure the background file exists
if [ ! -r "${BK_FILE}" ]; then
  echo "ERROR: ${BK_FILE} does not exist!"
  exit 1
fi


# Make sure OBS_ROOT is defined and exists
if [ ! "${OBS_ROOT}" ]; then
  echo "ERROR: \$OBS_ROOT is not defined!"
  exit 1
fi
if [ ! -d "${OBS_ROOT}" ]; then
  echo "ERROR: OBS_ROOT directory '${OBS_ROOT}' does not exist!"
  exit 1
fi


# Set the path to the GSI static files
if [ ! "${FIX_ROOT}" ]; then
  echo "ERROR: \$FIX_ROOT is not defined!"
  exit 1
fi
if [ ! -d "${FIX_ROOT}" ]; then
  echo "ERROR: fix directory '${FIX_ROOT}' does not exist!"
  exit 1
fi


# Set the path to the CRTM coefficients
if [ ! "${CRTM_ROOT}" ]; then
  echo "ERROR: \$CRTM_ROOT is not defined!"
  exit 1
fi
if [ ! -d "${CRTM_ROOT}" ]; then
  echo "ERROR: fix directory '${CRTM_ROOT}' does not exist!"
  exit 1
fi




# Make sure the GSI executable exists
if [ ! -x "${GSI_EXE}" ]; then
  echo "ERROR: ${GSI_EXE} does not exist!"
  exit 1
fi


# Check to make sure the number of processors for running GSI was specified
if [ -z "${GSIPROC}" ]; then
  echo "ERROR: The variable $GSIPROC must be set to contain the number of processors to run GSI"
  exit 1
fi


#
##################################################################################
# Create the ram work directory and cd into it


workdir=${WORK_ROOT}
echo " Create working directory:" ${workdir}


if [ -d "${workdir}" ]; then
  rm -rf ${workdir}
fi
mkdir -p ${workdir}
cd ${workdir}


#
##################################################################################


echo " Copy GSI executable, background file, and link observation bufr to working directory"


# Save a copy of the GSI executable in the workdir
cp ${GSI_EXE} gsi.exe


# Bring over background field (it's modified by GSI so we can't link to it)
cp ${BK_FILE} ./wrf_inout




# Link to the prepbufr data
ln -s ${PREPBUFR} ./prepbufr


# Link to the radiance data
# ln -s ${OBS_ROOT}/gdas1.t12z.1bamua.tm00.bufr_d amsuabufr
# ln -s ${OBS_ROOT}/gdas1.t12z.1bamub.tm00.bufr_d amsubbufr
# ln -s ${OBS_ROOT}/gdas1.t12z.1bhrs3.tm00.bufr_d hirs3bufr
# ln -s ${OBS_ROOT}/gdas1.t12z.1bhrs4.tm00.bufr_d hirs4bufr
# ln -s ${OBS_ROOT}/gdas1.t12z.1bmhs.tm00.bufr_d mhsbufr
# ln -s ${OBS_ROOT}/gdas1.t12z.gpsro.tm00.bufr_d gpsrobufr
#
##################################################################################


echo " Copy fixed files and link CRTM coefficient files to working directory"


# Set fixed files
#   berror   = forecast model background error statistics
#   specoef  = CRTM spectral coefficients
#   trncoef  = CRTM transmittance coefficients
#   emiscoef = CRTM coefficients for IR sea surface emissivity model
#   aerocoef = CRTM coefficients for aerosol effects
#   cldcoef  = CRTM coefficients for cloud effects
#   satinfo  = text file with information about assimilation of brightness temperatures
#   satangl  = angle dependent bias correction file (fixed in time)
#   pcpinfo  = text file with information about assimilation of prepcipitation rates
#   ozinfo   = text file with information about assimilation of ozone data
#   errtable = text file with obs error for conventional data (regional only)
#   convinfo = text file with information about assimilation of conventional data
#   bufrtable= text file ONLY needed for single obs test (oneobstest=.true.)
#   bftab_sst= bufr table for sst ONLY needed for sst retrieval (retrieval=.true.)


if [ ${bkcv_option} = GLOBAL ] ; then
  echo ' Use global background error covariance'
  BERROR=${FIX_ROOT}/${BYTE_ORDER}/nam_glb_berror.f77.gcv
  OBERROR=${FIX_ROOT}/prepobs_errtable.global
  if [ ${bk_core} = NMM ] ; then
     ANAVINFO=${FIX_ROOT}/anavinfo_ndas_netcdf_glbe
  else
    ANAVINFO=${FIX_ROOT}/anavinfo_arw_netcdf_glbe
  fi
else
  echo ' Use NAM background error covariance'
  BERROR=${FIX_ROOT}/${BYTE_ORDER}/nam_nmmstat_na.gcv
  OBERROR=${FIX_ROOT}/nam_errtable.r3dv
  if [ ${bk_core} = NMM ] ; then
     ANAVINFO=${FIX_ROOT}/anavinfo_ndas_netcdf
  else
     ANAVINFO=${FIX_ROOT}/anavinfo_arw_netcdf
  fi
fi


SATANGL=${FIX_ROOT}/global_satangbias.txt
SATINFO=${FIX_ROOT}/global_satinfo.txt
CONVINFO=${FIX_ROOT}/global_convinfo.txt
OZINFO=${FIX_ROOT}/global_ozinfo.txt
PCPINFO=${FIX_ROOT}/global_pcpinfo.txt


#  copy Fixed fields to working directory
cp $ANAVINFO anavinfo
cp $BERROR   berror_stats
cp $SATANGL  satbias_angle
cp $SATINFO  satinfo
cp $CONVINFO convinfo
cp $OZINFO   ozinfo
cp $PCPINFO  pcpinfo
cp $OBERROR  errtable
#
#    # CRTM Spectral and Transmittance coefficients
CRTM_ROOT_ORDER=${CRTM_ROOT}/${BYTE_ORDER}
emiscoef_IRwater=${CRTM_ROOT_ORDER}/Nalli.IRwater.EmisCoeff.bin
emiscoef_IRice=${CRTM_ROOT_ORDER}/NPOESS.IRice.EmisCoeff.bin
emiscoef_IRland=${CRTM_ROOT_ORDER}/NPOESS.IRland.EmisCoeff.bin
emiscoef_IRsnow=${CRTM_ROOT_ORDER}/NPOESS.IRsnow.EmisCoeff.bin
emiscoef_VISice=${CRTM_ROOT_ORDER}/NPOESS.VISice.EmisCoeff.bin
emiscoef_VISland=${CRTM_ROOT_ORDER}/NPOESS.VISland.EmisCoeff.bin
emiscoef_VISsnow=${CRTM_ROOT_ORDER}/NPOESS.VISsnow.EmisCoeff.bin
emiscoef_VISwater=${CRTM_ROOT_ORDER}/NPOESS.VISwater.EmisCoeff.bin
emiscoef_MWwater=${CRTM_ROOT_ORDER}/FASTEM5.MWwater.EmisCoeff.bin
aercoef=${CRTM_ROOT_ORDER}/AerosolCoeff.bin
cldcoef=${CRTM_ROOT_ORDER}/CloudCoeff.bin


ln -s $emiscoef_IRwater ./Nalli.IRwater.EmisCoeff.bin
ln -s $emiscoef_IRice ./NPOESS.IRice.EmisCoeff.bin
ln -s $emiscoef_IRsnow ./NPOESS.IRsnow.EmisCoeff.bin
ln -s $emiscoef_IRland ./NPOESS.IRland.EmisCoeff.bin
ln -s $emiscoef_VISice ./NPOESS.VISice.EmisCoeff.bin
ln -s $emiscoef_VISland ./NPOESS.VISland.EmisCoeff.bin
ln -s $emiscoef_VISsnow ./NPOESS.VISsnow.EmisCoeff.bin
ln -s $emiscoef_VISwater ./NPOESS.VISwater.EmisCoeff.bin
ln -s $emiscoef_MWwater ./FASTEM5.MWwater.EmisCoeff.bin
ln -s $aercoef  ./AerosolCoeff.bin
ln -s $cldcoef  ./CloudCoeff.bin
# Copy CRTM coefficient files based on entries in satinfo file
for file in `awk '{if($1!~"!"){print $1}}' ./satinfo | sort | uniq` ;do
   ln -s ${CRTM_ROOT_ORDER}/${file}.SpcCoeff.bin ./
   ln -s ${CRTM_ROOT_ORDER}/${file}.TauCoeff.bin ./
done


# Only need this file for single obs test
bufrtable=${FIX_ROOT}/prepobs_prep.bufrtable
cp $bufrtable ./prepobs_prep.bufrtable


# for satellite bias correction
cp ${FIX_ROOT}/sample.satbias ./satbias_in


#
##################################################################################
# Set some parameters for use by the GSI executable and to build the namelist
echo " Build the namelist "


if [ ${bkcv_option} = GLOBAL ] ; then
#   as_op='0.6,0.6,0.75,0.75,0.75,0.75,1.0,1.0'
   vs_op='0.7,'
   hzscl_op='1.7,0.8,0.5,'
else
#   as_op='1.0,1.0,0.5 ,0.7,0.7,0.5,1.0,1.0,'
   vs_op='1.0,'
   hzscl_op='0.373,0.746,1.50,'
fi


if [ ${bk_core} = NMM ] ; then
   bk_core_arw='.false.'
   bk_core_nmm='.true.'
else
   bk_core_arw='.true.'
   bk_core_nmm='.false.'
fi


# Build the GSI namelist on-the-fly
cat > gsiparm.anl << EOF
&SETUP
   miter=2,niter(1)=10,niter(2)=10,
   write_diag(1)=.true.,write_diag(2)=.false.,write_diag(3)=.true.,
   gencode=78,qoption=2,
   factqmin=0.0,factqmax=0.0,
   ndat=87,iguess=-1,
   oneobtest=.true.,retrieval=.false.,
   nhr_assimilation=3,l_foto=.false.,
   use_pbl=.false.,
/
&GRIDOPTS
   JCAP=62,JCAP_B=62,NLAT=60,NLON=60,nsig=60,regional=.true.,
   wrf_nmm_regional=${bk_core_nmm},wrf_mass_regional=${bk_core_arw},
   diagnostic_reg=.false.,
   filled_grid=.false.,half_grid=.true.,netcdf=.true.,
/
&BKGERR
   vs=${vs_op}
   hzscl=${hzscl_op}
   bw=0.,fstat=.true.,
/
&ANBKGERR
/
&JCOPTS
/
&STRONGOPTS
/
&OBSQC
   dfact=0.75,dfact1=3.0,noiqc=.false.,c_varqc=0.02,vadfile='prepbufr',
/
&OBS_INPUT
   dmesh(1)=120.0,dmesh(2)=60.0,dmesh(3)=60.0,dmesh(4)=60.0,dmesh(5)=120,time_window_max=1.5,
   dfile(01)='prepbufr',  dtype(01)='ps',        dplat(01)=' ',       dsis(01)='ps',                 dval(01)=1.0, dthin(01)=0, dsfcalc(01)=0,
   dfile(02)='prepbufr'   dtype(02)='t',         dplat(02)=' ',       dsis(02)='t',                  dval(02)=1.0, dthin(02)=0, dsfcalc(02)=0,
#************************************省略******************************************************************
   dfile(87)='gsnd1bufr', dtype(87)='sndrd4',    dplat(87)='g15',     dsis(87)='sndrD4_g15',         dval(87)=0.0, dthin(87)=1, dsfcalc(87)=0,
/
&SUPEROB_RADAR
   del_azimuth=5.,del_elev=.25,del_range=5000.,del_time=.5,elev_angle_max=5.,minnum=50,range_max=100000.,
   l2superob_only=.false.,
/
&LAG_DATA
/
&HYBRID_ENSEMBLE
   l_hyb_ens=.false.,
/
&RAPIDREFRESH_CLDSURF
/
&CHEM
/
&SINGLEOB_TEST
   maginnov=1.0,magoberr=0.8,oneob_type='t',
   oblat=45.,oblon=260.,obpres=500.,obdattim=${ANAL_TIME},
   obhourset=0.,
/


EOF


#
###################################################
#  run  GSI
###################################################
echo ' Run GSI with' ${bk_core} 'background'


case $ARCH in
   'IBM_LSF')
      ${RUN_COMMAND} ./gsi.exe < gsiparm.anl > stdout 2>&1  ;;


   * )
      ${RUN_COMMAND} ./gsi.exe > stdout 2>&1  ;;
esac
##################################################################
#  run time error check
##################################################################
error=$?


if [ ${error} -ne 0 ]; then
  echo "ERROR: ${GSI} crashed  Exit status=${error}"
  exit ${error}
fi
#
##################################################################
#
#   GSI updating satbias_in
#
# GSI updating satbias_in (only for cycling assimilation)


# Copy the output to more understandable names
ln -s stdout      stdout.anl.${ANAL_TIME}
ln -s wrf_inout   wrfanl.${ANAL_TIME}
ln -s fort.201    fit_p1.${ANAL_TIME}
ln -s fort.202    fit_w1.${ANAL_TIME}
ln -s fort.203    fit_t1.${ANAL_TIME}
ln -s fort.204    fit_q1.${ANAL_TIME}
ln -s fort.207    fit_rad1.${ANAL_TIME}


# Loop over first and last outer loops to generate innovation
# diagnostic files for indicated observation types (groups)
#
# NOTE:  Since we set miter=2 in GSI namelist SETUP, outer
#        loop 03 will contain innovations with respect to
#        the analysis.  Creation of o-a innovation files
#        is triggered by write_diag(3)=.true.  The setting
#        write_diag(1)=.true. turns on creation of o-g
#        innovation files.
#


loops="01 03"
for loop in $loops; do


case $loop in
  01) string=ges;;
  03) string=anl;;
   *) string=$loop;;
esac


#  Collect diagnostic files for obs types (groups) below
   listall="conv amsua_metop-a mhs_metop-a hirs4_metop-a hirs2_n14 msu_n14 \
          sndr_g08 sndr_g10 sndr_g12 sndr_g08_prep sndr_g10_prep sndr_g12_prep \
          sndrd1_g08 sndrd2_g08 sndrd3_g08 sndrd4_g08 sndrd1_g10 sndrd2_g10 \
          sndrd3_g10 sndrd4_g10 sndrd1_g12 sndrd2_g12 sndrd3_g12 sndrd4_g12 \
          hirs3_n15 hirs3_n16 hirs3_n17 amsua_n15 amsua_n16 amsua_n17 \
          amsub_n15 amsub_n16 amsub_n17 hsb_aqua airs_aqua amsua_aqua \
          goes_img_g08 goes_img_g10 goes_img_g11 goes_img_g12 \
          pcp_ssmi_dmsp pcp_tmi_trmm sbuv2_n16 sbuv2_n17 sbuv2_n18 \
          omi_aura ssmi_f13 ssmi_f14 ssmi_f15 hirs4_n18 amsua_n18 mhs_n18 \
          amsre_low_aqua amsre_mid_aqua amsre_hig_aqua ssmis_las_f16 \
          ssmis_uas_f16 ssmis_img_f16 ssmis_env_f16"
   for type in $listall; do
      count=0
      if [[ -f pe0000.${type}_${loop} ]]; then
         count=`ls pe*${type}_${loop}* | wc -l`
      fi
      if [[ $count -gt 0 ]]; then
         cat pe*${type}_${loop}* > diag_${type}_${string}.${ANAL_TIME}
      fi
   done
done


#  Clean working directory to save only important files
ls -l * > list_run_directory
if [ ${if_clean} = clean ]; then
  echo ' Clean working directory after GSI run'
  rm -f *Coeff.bin     # all CRTM coefficient files
  rm -f pe0*           # diag files on each processor
  rm -f obs_input.*    # observation middle files
  rm -f siganl sigf03  # background middle files
  rm -f fsize_*        # delete temperal file for bufr size
fi


exit 0
3.mpi运行报错为：

+ echo ' Run GSI with' ARW background
Run GSI with ARW background
+ mpiexec.hydra -machinefile /SWNEW/TEST/comGSI_v3.3/run/hosts -np 6 ./gsi.exe
+ 1> stdout 2>& 1
+ error=174
+ [ 174 -ne 0 ]
+ echo 'ERROR:  crashed  Exit status=174'
ERROR:  crashed  Exit status=174
4.stdout最后几行为：
GLBSOI: jiter,jiterstart,jiterlast,jiterend=           1           1
           1           1
GENSTATS_GPS:  no profiles to process (nprof_gfs=           0 ), EXIT routine
SETUPALL:,obstype,isis,nreal,nchanl=t         t                             24
           0
Begin Jo table outer loop
    Observation Type           Nobs                        Jo        Jo/n
temperature                       1    1.5625000000000000E+00       1.562
                               Nobs                        Jo        Jo/n
           Jo Global              1    1.5625000000000000E+00       1.562
End Jo table outer loop
GLBSOI:  START pcgsoi jiter=           1
forrtl: severe (174): SIGSEGV, segmentation fault occurred
Image              PC                Routine            Line        Source
gsi.exe            00000000017B4799  Unknown               Unknown  Unknown
gsi.exe            00000000017B306E  Unknown               Unknown  Unknown
gsi.exe            00000000017402A2  Unknown               Unknown  Unknown
gsi.exe            00000000016C2AB8  Unknown               Unknown  Unknown
gsi.exe            00000000016C79BB  Unknown               Unknown  Unknown
libpthread.so.0    0000003213C0F710  Unknown               Unknown  Unknown
gsi.exe            0000000000D5D7AB  frfhvo_                    63  smoothzrf.f90
gsi.exe            0000000000EBBF27  Unknown               Unknown  Unknown
libiomp5.so        00002B31CB493233  Unknown               Unknown  Unknown
forrtl: severe (174): SIGSEGV, segmentation fault occurred

大马哈

呀 这个我好久都不弄了 记得会输出好多的诊断文件的，不知道是不是可以从那边看看什么问题  原谅我这个酱油吧

哞哞三年半

大马哈 发表于 2017-4-25 09:24
呀 这个我好久都不弄了  记得会输出好多的诊断文件的，不知道是不是可以从那边看看什么问题  原 ...

亲！你用intel编译器编过GSI3.3吗？ 我在两台机器上，用pgi编译器就能编译且测试官网case2成功；用icc/ifort编译能生成gsi.exe，但是测试官网数据就如下错误：
C:\Users\11363\Desktop\GSI Assimilation\QQ图片20170425170715.png

kazami2003

你可以先try "ulimit -s unlimited", 然後在run一次。

哞哞三年半

kazami2003 发表于 2017-4-28 06:58
你可以先try "ulimit -s unlimited", 然後在run一次。

感谢~是改了stack size，依旧跑不通。结果，，，，换成15.0的intel编译器就ok了，官网例子跑通了。

边界

话说这些软件，如ifort在哪搞到的？

clarawho

哞哞三年半 发表于 2017-5-19 13:43
感谢~是改了stack size，依旧跑不通。结果，，，，换成15.0的intel编译器就ok了，官网例子跑通了。

请问怎么换成15.0的intel编译器？谢谢{:eb348:}

2830882730

我也遇到了这个问题，请问楼主怎么知道自己版本gsi适配多少版本的intel编译器？我现在官网的case都能跑的通，自己的同化跟你报一样的错误，请问是intel编译器的问题吗