=====Overview=====
DBFS or database file system is another twerk of oracle. Basically it is similiar to the NFS in a sense that it can be mounted and dismounted, but also, it can be registered in a GRID infrastructure.
That will allow it to failvoer in case of failure of one of the instances, so let's see how it is done.


=====Creation=====
To create DBFS we need couple things:

  - Edit FUSE config (that is the config file for the FUSE package needed for the DBFS.
  - Create folder (owned under oracle user and dba group)
  - Create a tablespace which will host the DBFS 
  - Create user with the necessary permission
  - Create the DBFS env on the database
  - Configure the libraries

So let's get started:

First, we have to edit the configuration for FUSE:
<Code:bash|Edit FUSE config>
[root@ol6-121-rac2 ~]# vi /etc/fuse.conf 
user_allow_other
!wq
</Code>

Then we have to create the directory:

<Code:bash|Create directory>
[root@ol6-121-rac1 ~]# mkdir -p /oracle/dbfs_direct
[root@ol6-121-rac1 ~]# chown oracle:dba /oracle/dbfs_direct
[root@ol6-121-rac1 ~]# ls –ld /oracle/dbfs_direct
drwxr-xr-x 2 oracle dba 4096 Nov  3 02:26 /oracle/dbfs_direct/
[root@ol6-121-rac1 ~]#
</Code>

Optionally we can configure NOCACHE for the LOB of the DBFS table:

<Code:bash|Modify DBFS table>
[root@ol6-121-rac1 ~]# echo $ORACLE_SID
OGGRAC1
[root@ol6-121-rac1 ~]# sqlplus / as sysdba
SQL*Plus: Release 12.1.0.2.0 Production on Mon Mar 30 22:20:19
2015
Copyright (c) 1982, 2014, Oracle. All rights reserved.
Connected to:
Oracle Database 12c Enterprise Edition Release 12.1.0.2.0 -
64bit Production
With the Partitioning, Real Application Clusters, Automatic
Storage Management, OLAP,
Advanced Analytics and Real Application Testing options
SQL> ALTER TABLE DBFS_USER.FS1 MODIFY LOB (FILEDATA) (NOCACHE
LOGGING);
Table altered.
SQL> column owner format a10
SQL> column table_name format a15
SQL> column segment_name format a15
SQL> SELECT owner,table_name,segment_name,logging,cache FROM
dba_lobs WHERE tablespace_name='DBFS_TS';
OWNER TABLE_NAME SEGMENT_NAME LOGGING CACHE
---------- ------------- -------------------- ------- ----------
DBFS_USER FS1 LOB_SFS$_FST_1 YES NO
SQL> exit
Disconnected from Oracle Database 12c Enterprise Edition Release
12.1.0.2.0 - 64bit Production
With the Partitioning, Real Application Clusters, Automatic
Storage Management, OLAP,
Advanced Analytics and Real Application Testing options
[root@ol6-121-rac1 ~]#
</Code>

After that, we have to create the Tablespace, user and the environment:

<Code:bash|Configure DBFS on the database>
[oracle@ol6-121-rac1 ~]$ sqlplus

SQL*Plus: Release 12.1.0.2.0 Production on Tue Nov 3 02:30:01 2020

Copyright (c) 1982, 2014, Oracle.  All rights reserved.

Enter user-name: / as sysdba

Connected to:
Oracle Database 12c Enterprise Edition Release 12.1.0.2.0 - 64bit Production
With the Partitioning, Real Application Clusters, Automatic Storage Management, OLAP,
Advanced Analytics and Real Application Testing options

SQL> create bigfile tablespace dbfs_ts datafile '+DATA' size 1024M autoextend on next 100M maxsize 3G NOLOGGING EXTENT MANAGEMENT LOCAL AUTOALLOCATE SEGMENT SPACE MANAGEMENT AUTO;

Tablespace created.

SQL> create user dbfs_user identified by oracle default tablespace dbfs_ts quota unlimited on dbfs_ts;

User created.

SQL> grant create session, create table, create view, create procedure, dbfs_role to dbfs_user;

Grant succeeded.

SQL> exit
Disconnected from Oracle Database 12c Enterprise Edition Release 12.1.0.2.0 - 64bit Production
With the Partitioning, Real Application Clusters, Automatic Storage Management, OLAP,
Advanced Analytics and Real Application Testing options
[oracle@ol6-121-rac1 ~]$ cd /u01/app/oracle/product/12.1.0.2/db_1/rdbms/admin/

[oracle@ol6-121-rac1 admin]$ sqlplus dbfs_user/oracle
SQL*Plus: Release 12.1.0.2.0 Production on Tue Nov 3 02:41:53 2020

Copyright (c) 1982, 2014, Oracle.  All rights reserved.


Connected to:
Oracle Database 12c Enterprise Edition Release 12.1.0.2.0 - 64bit Production
With the Partitioning, Real Application Clusters, Automatic Storage Management, OLAP,
Advanced Analytics and Real Application Testing options

SQL> start dbfs_create_filesystem dbfs_ts FS1
No errors.
--------
CREATE STORE:
begin dbms_dbfs_sfs.createFilesystem(store_name => 'FS1', tbl_name => 'FS1',
tbl_tbs => 'dbfs_ts', lob_tbs => 'dbfs_ts', do_partition => false, partition_key
=> 1, do_compress => false, compression => '', do_dedup => false, do_encrypt =>
false); end;
--------
REGISTER STORE:
begin dbms_dbfs_content.registerStore(store_name=> 'FS1', provider_name =>
'sample1', provider_package => 'dbms_dbfs_sfs'); end;
--------
MOUNT STORE:
begin dbms_dbfs_content.mountStore(store_name=>'FS1', store_mount=>'FS1'); end;
--------
CHMOD STORE:
declare m integer; begin m := dbms_fuse.fs_chmod('/FS1', 16895); end;
No errors.
SQL> 
</Code>

Configure the libraries:

<Code:bash|Configure Libraries>
[oracle@ol6-121-rac1 admin]$ su -
Password: 
[root@ol6-121-rac1 ~]# echo "/usr/local/lib" >> /etc/ld.so.conf.d/usr_local_lib.conf
[root@ol6-121-rac1 ~]# cat /etc/ld.so.conf.d/usr_local_lib.conf
/usr/local/lib
[root@ol6-121-rac1 ~]# cd /usr/local/lib
[root@ol6-121-rac1 lib]# pwd
/usr/local/lib
[root@ol6-121-rac1 lib]# ln -s /u01/app/12.1.0.2/grid/lib/libclntsh.so.12.1
[root@ol6-121-rac1 lib]# ln -s /u01/app/12.1.0.2/grid/lib/libnnz12.so
[root@ol6-121-rac1 lib]# ln -s /lib64/libfuse.so.2 libfuse.so
[root@ol6-121-rac1 lib]# ls -l
total 0
lrwxrwxrwx 1 root root 44 Nov  3 03:16 libclntsh.so.12.1 -> /u01/app/12.1.0.2/grid/lib/libclntsh.so.12.1
lrwxrwxrwx 1 root root 19 Nov  3 03:17 libfuse.so -> /lib64/libfuse.so.2
lrwxrwxrwx 1 root root 38 Nov  3 03:17 libnnz12.so -> /u01/app/12.1.0.2/grid/lib/libnnz12.so
[root@ol6-121-rac1 lib]#
[root@ol6-121-rac1 lib]# ldconfig
[root@ol6-121-rac1 lib]# ln -s /u01/app/12.1.0.2/grid/bin/dbfs_client /sbin/mount.dbfs
[root@ol6-121-rac1 lib]# ls -l /sbin/mount.dbfs
lrwxrwxrwx 1 root root 38 Nov  3 03:17 /sbin/mount.dbfs -> /u01/app/12.1.0.2/grid/bin/dbfs_client
[root@ol6-121-rac1 lib]# chmod +x /bin/fusermount
[root@ol6-121-rac1 lib]# cd /u01/app/12.1.0.2/grid/crs/script
[root@ol6-121-rac1 script]# cp /media/sf_install/mount-dbfs.sh .
[root@ol6-121-rac1 script]# 
[root@ol6-121-rac1 script]# chown oracle.dba mount-dbfs.sh
[root@ol6-121-rac1 script]# chmod 750 mount-dbfs.sh
[root@ol6-121-rac1 script]# ls -l
total 16
-rwxr-x--- 1 oracle dba 14083 Nov  3 03:29 mount-dbfs.sh
</Code>


=====Management=====
The management of DBFS, includes, stop/start and registartion in the Grid Infra:

====Start/Stop====
<Code:bash|Start DBFS>
--Mount
[oracle@ol6-121-rac1 admin]$ id
uid=54321(oracle) gid=54321(oinstall) groups=54321(oinstall),492(vboxsf),54322(dba)
[oracle@ol6-121-rac1 admin]$ /u01/app/12.1.0.2/grid/crs/script/mount-dbfs.sh start
mount-dbfs.sh mounting DBFS at /oracle/dbfs_direct from database OGGRAC
ORACLE_SID is OGGRAC1
spawning dbfs_client command using SID OGGRAC1
nohup: redirecting stderr to stdout
Start -- ONLINE
[oracle@ol6-121-rac1 admin]$ 
[oracle@ol6-121-rac1 admin]$ /u01/app/12.1.0.2/grid/crs/script/mount-dbfs.sh status
Checking status now
Check -- ONLINE
[oracle@ol6-121-rac1 admin]$ df -h
Filesystem            Size  Used Avail Use% Mounted on
/dev/mapper/vg_ol6121rac1-lv_root
                       46G   19G   25G  43% /
tmpfs                 1.9G  628M  1.2G  34% /dev/shm
/dev/sda1             477M   66M  382M  15% /boot
install               402G  190G  212G  48% /media/sf_install
dbfs-dbfs_user@:/     3.0G  120K  3.0G   1% /oracle/dbfs_direct
[oracle@ol6-121-rac1 admin]$ 

--Unmount
[oracle@ol6-121-rac1 admin]$ /u01/app/12.1.0.2/grid/crs/script/mount-dbfs.sh stop
unmounting DBFS from /oracle/dbfs_direct
umounting the filesystem using '/bin/fusermount -u /oracle/dbfs_direct'
Stop - stopped, now not mounted
[oracle@ol6-121-rac1 admin]$ df -h
Filesystem            Size  Used Avail Use% Mounted on
/dev/mapper/vg_ol6121rac1-lv_root
                       46G   19G   25G  43% /
tmpfs                 1.9G  628M  1.2G  34% /dev/shm
/dev/sda1             477M   66M  382M  15% /boot
install               402G  190G  212G  48% /media/sf_install
[oracle@ol6-121-rac1 admin]$ 
</Code>

====Register DBFS into Grid Infra====
We can register the DBFS into Grid infra and though allow failover if the instance crashes:

<Code:bash|Register DBFS into Grid Infra>
[OS prompt]$ cp /media/sf_install/add-dbfs-resource.sh ~
[OS prompt]$ cd ~
[OS prompt]$ chmod +x add-dbfs-resource.sh
[OS prompt]$ cat add-dbfs-resource.sh
[oracle@ol6-121-rac1 ~]$ ./add-dbfs-resource.sh
[oracle@ol6-121-rac1 ~]$ grid_env
[oracle@ol6-121-rac1 ~]$ crsctl stat res dbfs_mount
NAME=dbfs_mount
TYPE=local_resource
TARGET=OFFLINE, OFFLINE
STATE=OFFLINE, OFFLINE
[oracle@ol6-121-rac1 ~]$ crsctl start res dbfs_mount
CRS-2672: Attempting to start 'dbfs_mount' on 'ol6-121-rac1'
CRS-2672: Attempting to start 'dbfs_mount' on 'ol6-121-rac2'
CRS-2676: Start of 'dbfs_mount' on 'ol6-121-rac2' succeeded
CRS-2676: Start of 'dbfs_mount' on 'ol6-121-rac1' succeeded
[oracle@ol6-121-rac1 ~]$ crsctl stat res dbfs_mount
NAME=dbfs_mount
TYPE=local_resource
TARGET=ONLINE                , ONLINE
STATE=ONLINE on ol6-121-rac1, ONLINE on ol6-121-rac2
</Code>



=====Appendix=====
Below you can find the script we have used.

<Code:bash|mount-dbfs.sh>
#!/bin/bash

### This script is from Note 1054431.1, ensure you have the latest version
### Note 1054431.1 provides information about the setup required to use this script

### updated 23-DEC-2014

###########################################
### Everyone must set these values
###########################################
### Database name for the DBFS repository as used in "srvctl status database -d $DBNAME"
DBNAME=OGGRAC

### Mount point where DBFS should be mounted 
MOUNT_POINT=/oracle/dbfs_direct

### Username of the DBFS repository owner in database $DBNAME
DBFS_USER=dbfs_user

### RDBMS ORACLE_HOME directory path
ORACLE_HOME=/u01/app/oracle/product/12.1.0.2/db_1

### Syslog facility name (default user)
### Changed default from local3 to user for Solaris default support on 17-FEB-2012
### This will allow us to log messages to the syslog 
###  (/var/log/messages on Linux, /var/adm/messages on Solaris)
LOGGER_FACILITY=user

### mount options for dbfs_client; these are used for both wallet and non-wallet mounting
MOUNT_OPTIONS=allow_other,direct_io
### if tracing is required, maybe consider parameter like the example below
### following example is commented out, only uncomment if directed by Oracle Support
### fix_control=32 added per bug 13340960 to allow async statfs response
#MOUNT_OPTIONS=allow_other,direct_io,fix_control=32,trace_level=1,trace_file=/tmp/dbfs_client_trace.$$.log,trace_size=100

### PERL_ALARM_TIMEOUT is number of seconds to wait for response from status command.
### After this, if no respnose, the script will run clean.
### NOTE: If this is longer than the clusterware check interval, bad things may happen.
###       Adjust the CHECK_INTERVAL to ensure it is at least 2x as long as PERL_ALARM_TIMEOUT.
###       Example:
###       $ crsctl status res dbfs_mount -p|grep ^CHECK      
###       CHECK_INTERVAL=30
###       $ crsctl modify res dbfs_mount -attr "CHECK_INTERVAL=32"
###       $ crsctl status res dbfs_mount -p|grep ^CHECK
###       CHECK_INTERVAL=32
PERL_ALARM_TIMEOUT=14

###########################################
### If using password-based authentication, set these
###########################################
### This is the plain text password for the DBFS_USER user
DBFS_PASSWD=oracle

### The file used to temporarily store the DBFS_PASSWD so dbfs_client can read it
### This file is removed immediately after it is read by dbfs_client
### The actual filename used will have the PID appended to the name for uniqueness
### This variable should be a full pathname including a directory and the first part of a filename.
DBFS_PWDFILE_BASE=/tmp/.dbfs-passwd.txt

###########################################
### If using wallet-based authentication, modify these
###########################################
### WALLET should be true if using a wallet, otherwise, false
WALLET=false

### TNS_ADMIN is the directory containing tnsnames.ora and sqlnet.ora used by DBFS
TNS_ADMIN=/export/home/oracle/dbfs/tnsadmin

### TNS alias used for mounting with wallets
DBFS_LOCAL_TNSALIAS=fsdb.local

###########################################
### No editing is required below this point
###########################################
### determine platform
UNAME_S=`uname -s`
if   [ $UNAME_S = 'Linux' ]; then LINUX=1; SOLARIS=0; 
elif [ $UNAME_S = 'SunOS' ]; then LINUX=0; SOLARIS=1; 
fi

GREP=/bin/grep
AWK=/bin/awk
ECHO=/bin/echo
LOGGER="/bin/logger -t DBFS_${MOUNT_POINT}"
RMF='/bin/rm -f'
TOUCH=/bin/touch
CHMOD=/bin/chmod
PS=/bin/ps
SLEEP=/bin/sleep
KILL=/bin/kill
BASENAME=/bin/basename
STAT=/usr/bin/stat
ID=/usr/bin/id
WC=/usr/bin/wc
SRVCTL=$ORACLE_HOME/bin/srvctl
DBFS_CLIENT=$ORACLE_HOME/bin/dbfs_client
HN=/bin/hostname
PERL=/usr/bin/perl
MOUNT=/bin/mount
### ensure messages are displayed in English for pattern matching
LANG=en_US.UTF-8
NLS_LANG=American_America.AL32UTF8
NUMACTL=/usr/bin/numactl
RPMCTL=/bin/rpm

if [ -z "$STATUS_TIMEOUT" ]; then STATUS_TIMEOUT=0; fi

if [ $LINUX -eq 1 ]; then
  MOUNT=/bin/mount
  XARGS='/usr/bin/xargs -r'
  FUSERMOUNT=/bin/fusermount
  LD_LIBRARY_PATH=$ORACLE_HOME/lib:/lib64
elif [ $SOLARIS -eq 1 ]; then
  MOUNT=/sbin/mount
  XARGS=/usr/bin/xargs
  UMOUNT=/usr/sbin/umount
  LD_LIBRARY_PATH=$ORACLE_HOME/lib:$ORACLE_HOME/rdbms/lib:/usr/lib:/lib
fi

DBFS_PWDFILE=$DBFS_PWDFILE_BASE.$$

export ORACLE_HOME LD_LIBRARY_PATH TNS_ADMIN
export STAT MOUNT_POINT PERL_ALARM_TIMEOUT SOLARIS LINUX
export PATH=$ORACLE_HOME/bin:$PATH
export STATUS_TIMEOUT
export LANG NLS_LANG

logit () {
  ### type: info, error, debug
  type=$1
  msg=$2
  if [ "$type" = "info" ]; then
    $ECHO $msg
    $LOGGER -p ${LOGGER_FACILITY}.info "$msg"
  elif [ "$type" = "error" ]; then
    $ECHO $msg
    $LOGGER -p ${LOGGER_FACILITY}.error "$msg"
  elif [ "$type" = "debug" ]; then
    $ECHO $msg
    $LOGGER -p ${LOGGER_FACILITY}.debug "$msg"
  fi
}

### must not be root
if [ `$ID -u` -eq 0 ]; then
  logit error "Run this as the Oracle software owner, not root"
  exit 1
fi

### determine how we were called, derive location
SCRIPTPATH=$0
SCRIPTNAME=`$BASENAME $SCRIPTPATH`

echo $SCRIPTPATH | grep ^/ > /dev/null 2>&1
if [ $? -ne 0 ]; then 
  MYDIR=`pwd`
  SCRIPTPATH=${MYDIR}/${SCRIPTPATH}
fi

### must cd to a directory where the oracle owner can get CWD
cd /tmp

case "$1" in
'start')
  logit info "$SCRIPTNAME mounting DBFS at $MOUNT_POINT from database $DBNAME"

  ### check to see if it is already mounted
  $SCRIPTPATH status > /dev/null 2>&1
  if [ $? -eq 0 ]; then
    logit error "$MOUNT_POINT already mounted, use $SCRIPTNAME stop before attempting to start"
    $SCRIPTPATH status
    exit 1
  fi

  ### set the ORACLE_SID dynamically based on OCR info, if it is running
  export ORACLE_SID=$($SRVCTL status instance -d $DBNAME -n `$HN` | \
                      $GREP 'is running' | $AWK '{print $2}' )
  # if single instance, set ORACLE_SID based on below command instead
  if [ -z "$ORACLE_SID" ]; then
    export ORACLE_SID=$($SRVCTL config db -d $DBNAME | $GREP 'instance' | $AWK '{print $3}')
  fi
  logit info "ORACLE_SID is $ORACLE_SID"

  ### if there's no SID defined locally or it isn't running, stop
  if [ -z "$ORACLE_SID" -a "$WALLET" = 'false' ]; then
    logit error "No running ORACLE_SID available on this host, exiting"
    exit 2
  fi

  ### version comparison function, used in numa section
  # Compare string versions
  # Returns in stdout and in the err code
  # 0        - a equal b
  # 1        - a greater than b
  # 255(-1)  - a less than b
  # version should be in format 1.2.3.4.5
  # It might be a short version like 1.2.3 or 1.2.3.
  version_cmp ()
  {
    local -a al=(`echo $1 | sed -e 's/\./ /g'`)
    local -a bl=(`echo $2 | sed -e 's/\./ /g'`)

    local -i i=0
    for ((i=0; i < ${#al[@]}; i++)); do
      # ap is always non-empty. Gap is not possible in both arrays
      local ap=${al[$i]}
      local bp=${bl[$i]}
      # Only $ap defined. $a longer (bigger) then $b
      if [ -z "$bp" ] || [ $ap -gt $bp ]; then
        echo "1"
        return 1
      elif [ $ap -lt $bp ]; then
        echo "-1"
        return -1
      fi
    done
    # Check for the next part from @bl. It means $a shorter (smaller) then $b
    if [ -n "${bl[$i]}" ]; then
      echo "-1"
      return -1
    fi
    # Both arrays ended at the same time. They are equal
    echo "0"
    return 0
  }
  
  ### if numa system, update mount_options for bug 10004611
  RPMEXA=`$RPMCTL -q --queryformat '%{VERSION}' exadata-base`
  NUMASYS=`$NUMACTL --hardware | grep available: | cut -c1-18`
  ###Check if an X8 system and image greater than or equal to 11.2.3.3.0 to support numa mount option
  if [ "$NUMASYS" = 'available: 8 nodes' ]; then
    if [ `version_cmp $RPMEXA 11.2.3.3.0` -eq 0 ] || [ `version_cmp $RPMEXA 11.2.3.3.0` -eq 1 ]; then
      MOUNT_OPTIONS=$MOUNT_OPTIONS,numa
    else
      MOUNT_OPTIONS=$MOUNT_OPTIONS
    fi
  ###Check if an X5-2 system to support numa mount option
  elif [ "$NUMASYS" = 'available: 2 nodes' ]; then
    MOUNT_OPTIONS=$MOUNT_OPTIONS,numa
  ###Check for all other X2 systems, do not use numa option
  elif [ "$NUMASYS" = 'available: 1 nodes' ]; then
    MOUNT_OPTIONS=$MOUNT_OPTIONS
  ###Exit for errors or unexpected values, post in /var/log/messages
  else
    logit error "Unexpected numa value.  Exiting."
    logit error "Numa hardware value is: $NUMASYS"
    logit error "RPM version of Exadata base is: $RPMEXA"  
    exit 1
  fi

  ### if using password-based startup, use this
  if [ "$WALLET" = 'false' -a -n "$DBFS_PASSWD" ]; then
    $RMF $DBFS_PWDFILE
    if [ -f $DBFS_PWDFILE ]; then 
      logit error "please remove $DBFS_PWDFILE and try again"
      exit 1
    fi 

    $TOUCH $DBFS_PWDFILE
    $CHMOD 600 $DBFS_PWDFILE
    $ECHO $DBFS_PASSWD > $DBFS_PWDFILE

    logit info "spawning dbfs_client command using SID $ORACLE_SID"

    (nohup $DBFS_CLIENT ${DBFS_USER}@ -o $MOUNT_OPTIONS \
          $MOUNT_POINT < $DBFS_PWDFILE | $LOGGER -p ${LOGGER_FACILITY}.info 2>&1 & ) &

  elif [ "$WALLET" = true ]; then
    logit info "doing mount $MOUNT_POINT using SID $ORACLE_SID with wallet now"
    (nohup $DBFS_CLIENT /@${DBFS_LOCAL_TNSALIAS} -o $MOUNT_OPTIONS,wallet \
          $MOUNT_POINT | $LOGGER -p ${LOGGER_FACILITY}.info 2>&1 & ) &
  fi

  ### allow time for the mount table update before checking it
  $SLEEP 1
  ### set return code based on success of mounting
  $SCRIPTPATH status > /dev/null 2>&1
  if [ $? -eq 0 ]; then
    logit info "Start -- ONLINE"
    exit 0
  else
    logit info "Start -- OFFLINE"
    exit 1
  fi
  
  $RMF $DBFS_PWDFILE
  ;;

'stop')
  $SCRIPTPATH status > /dev/null
  if [ $? -eq 0 ]; then
    logit info "unmounting DBFS from $MOUNT_POINT"
    if [ $LINUX -eq 1 ]; then
      logit info "umounting the filesystem using '$FUSERMOUNT -u $MOUNT_POINT'"
      $FUSERMOUNT -u $MOUNT_POINT
    elif [ $SOLARIS -eq 1 ]; then
      logit info "umounting the filesystem using '$UMOUNT $MOUNT_POINT'"
      $UMOUNT $MOUNT_POINT > /dev/null 2>&1
    fi
    $SCRIPTPATH status > /dev/null
    if [ $? -eq 0 ]; then
      logit error "Stop - stopped, but still mounted, error"
      exit 1
    else
      logit info "Stop - stopped, now not mounted"
      exit 0
    fi
  else
    logit error "filesystem $MOUNT_POINT not currently mounted, no need to stop"
  fi
  ;;

'check'|'status')
  ### check to see if it is mounted
  ### fire off a short process in perl to do the check (need the alarm builtin)
  logit debug "Checking status now"
  $PERL <<'TOT'
    $timeout = $ENV{'PERL_ALARM_TIMEOUT'};
    $SIG{ALRM} = sub { 
      ### we have a problem and need to cleanup
      exit 3;
      die "timeout" ;
    };
    alarm $timeout;
    eval {
      $STATUSOUT=`$ENV{'STAT'} -f -c "%T" $ENV{'MOUNT_POINT'} 2>&1 `; 
      chomp($STATUSOUT);
      ### added fuseblk check for Linux 6 output
      if ( ( $ENV{'SOLARIS'} == 1 && $STATUSOUT eq 'uvfs' ) ||
           ( $ENV{'LINUX'} == 1   && $STATUSOUT eq 'UNKNOWN (0x65735546)' ) ||
           ( $ENV{'LINUX'} == 1   && $STATUSOUT eq 'fuseblk' ) ) {
        ### status is okay
        exit 0;
      } elsif ( $STATUSOUT =~ /Transport endpoint is not connected/ ) {
        ### we have a problem, need to clean up
        exit 2;
      } else {
        ### filesystem is offline
        exit 1;
      }
    };

TOT

  RC=$?
  ### process return codes from the perl block
  if [ $RC -eq 3 ]; then
    STATUS_TIMEOUT=$(( $STATUS_TIMEOUT + 1 ))
    logit error "Found timeout while checking status, cleaning mount automatically"
    $SCRIPTPATH clean
    logit debug "Check -- OFFLINE"
    exit 1
  elif [ $RC -eq 2 ]; then
    STATUS_TIMEOUT=$(( $STATUS_TIMEOUT + 1 ))
    logit error "Found error while checking status, cleaning mount automatically"
    $SCRIPTPATH clean
    logit debug "Check -- OFFLINE"
    exit 1
  elif [ $RC -eq 1 ]; then
    logit debug "Check -- OFFLINE"
    exit 1
  elif [ $RC -eq 0 ]; then
    logit debug "Check -- ONLINE"
    exit 0
  fi
  ;;

'restart')
  logit info "restarting DBFS" 
  $SCRIPTPATH stop
  $SLEEP 2
  $SCRIPTPATH start
  ;;

'clean'|'abort')
  logit info "cleaning up DBFS nicely using (fusermount -u|umount)"
  if [ $LINUX -eq 1 ]; then
    $FUSERMOUNT -u $MOUNT_POINT
  elif [ $SOLARIS -eq 1 ]; then
    $UMOUNT $MOUNT_POINT > /dev/null 2>&1
  fi
  $SLEEP 1
  FORCE_CLEANUP=0
  if [ $STATUS_TIMEOUT -gt 1 ]; then
    FORCE_CLEANUP=1
  else
    $SCRIPTPATH status > /dev/null 
    if [ $? -eq 0 ]; then FORCE_CLEANUP=1; fi
  fi
  if [ $FORCE_CLEANUP -eq 1 ]; then 
    logit error "tried (fusermount -u|umount), still mounted, now cleaning with (fusermount -u -z|umount -f) and kill"
    if [ $LINUX -eq 1 ]; then
      $FUSERMOUNT -u -z $MOUNT_POINT
    elif [ $SOLARIS -eq 1 ]; then
      echo "running umount -f now"
      $UMOUNT -f $MOUNT_POINT > /dev/null 2>&1
    fi
    if [ $LINUX -eq 1 ]; then
      PIDS=`$PS -ef | $GREP -w "$MOUNT_POINT" | $GREP dbfs_client| $GREP -v grep | \
        $AWK '{print $2}'`
      if [ -n "$PIDS" ]; then $KILL -9 $PIDS; fi
      PIDS=`$PS -ef | $GREP -w "$MOUNT_POINT" | $GREP mount.dbfs | $GREP -v grep | \
        $AWK '{print $2}'`
      if [ -n "$PIDS" ]; then $KILL -9 $PIDS; fi
    elif [ $SOLARIS -eq 1 ]; then
      PIDS=`$PS -ef | $GREP dbfs_client| $GREP -v grep | $AWK '{print $2}'`
      REALPIDS=' '
      for pid in $PIDS
      do
        ARGS=`pargs $pid`
        echo $ARGS | grep  "$MOUNT_POINT$"  > /dev/null
        RET=$?
        if [ $RET -eq 0 ]; then REALPIDS="$REALPIDS $pid"; fi
      done
      if [ -n "$REALPIDS" ]; then $KILL -9 $REALPIDS; fi
      ### do it a 2nd time to clean up others
      if [ -n "$REALPIDS" ]; then $KILL -9 $REALPIDS; fi
      PIDS=`$PS -ef | $GREP dbfs_client| $GREP -v grep | $AWK '{print $2}'`
      REALPIDS=' '
      for pid in $PIDS
      do
        ARGS=`pargs $pid`
        echo $ARGS | grep  "$MOUNT_POINT$"  > /dev/null
        RET=$?
        if [ $RET -eq 0 ]; then REALPIDS="$REALPIDS $pid"; fi
      done
      if [ -n "$REALPIDS" ]; then $KILL -9 $REALPIDS; fi

    fi
    exit 1
  fi
  ;;

*)
  $ECHO "Usage: $SCRIPTNAME { start | stop | check | status | restart | clean | abort }"
  ;;

esac
</Code>

<Code:bash|add-dbfs-resource.sh>
##### start script add-dbfs-resource.sh
#!/bin/bash
ACTION_SCRIPT=/u01/app/12.1.0.2/grid/crs/script/mount-dbfs.sh
RESNAME=dbfs_mount
DBNAME=OGGRAC
DBNAMEL=`echo $DBNAME | tr A-Z a-z`
ORACLE_HOME=/u01/app/12.1.0.2/grid
PATH=$ORACLE_HOME/bin:$PATH
export PATH ORACLE_HOME
crsctl add resource $RESNAME \
-type local_resource \
-attr "ACTION_SCRIPT=$ACTION_SCRIPT, \
CHECK_INTERVAL=30,RESTART_ATTEMPTS=10, \
START_DEPENDENCIES='hard(ora.$DBNAMEL.db)pullup(ora.$DBNAMEL.db)
',\
STOP_DEPENDENCIES='hard(ora.$DBNAMEL.db)',\
SCRIPT_TIMEOUT=300"
##### end script add-dbfs-resource.sh
</Code>