粘贴箱: update-mime-databaseのfdatasyncの呼び出し時間短縮による効率化(for linux)

格式

Diff

提交日期

2020-03-11 12:28

Publication Period

Unlimited

 --- a/update-mime-database.c
 +++ b/update-mime-database.c
 @@ -1,3 +1,5 @@
 +#define _GNU_SOURCE
 +
  #include <config.h>
  
  #define N_(x) x
 @@ -37,6 +39,15 @@
  #define NOGLOBS "__NOGLOBS__"
  #define NOMAGIC "__NOMAGIC__"
  
 +#ifdef __linux__
 +# include <linux/version.h>
 +# if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 5, 0)
 +#  include <linux/aio_abi.h>
 +#  include <sys/syscall.h>
 +#  include <sys/eventfd.h>
 +# endif
 +#endif
 +
  #ifndef PATH_SEPARATOR
  # ifdef _WIN32
  #  define PATH_SEPARATOR ";"
 @@ -168,12 +179,31 @@
  /* Lists enabled log levels */
  static GLogLevelFlags enabled_log_levels = G_LOG_LEVEL_ERROR | G_LOG_LEVEL_CRITICAL | G_LOG_LEVEL_WARNING;
  
 +static GHashTable *rename_set = NULL;
 +
  /* Static prototypes */
  static Magic *magic_new(xmlNode *node, Type *type, GError **error);
  static Match *match_new(void);
  
  static TreeMagic *tree_magic_new(xmlNode *node, Type *type, GError **error);
  
 +#ifdef __LINUX__AIO_ABI_H
 +static int io_setup(unsigned nr_events, aio_context_t *ctx_idp){
 +       return syscall(SYS_io_setup, nr_events, ctx_idp);
 +}
 +static int io_submit(aio_context_t ctx_id, long nr, struct iocb **iocbpp){
 +       return syscall(SYS_io_submit, ctx_id, nr, iocbpp);
 +}
 +static int io_getevents(aio_context_t ctx_id, long min_nr, long nr,
 +       struct io_event *events, struct timespec *timeout
 +){
 +       return syscall(SYS_io_getevents, ctx_id, min_nr, nr, events, timeout);
 +}
 +static int io_destroy(aio_context_t ctx_id){
 +       return syscall(SYS_io_destroy, ctx_id);
 +}
 +#endif
 +
  static void g_log_handler (const gchar   *log_domain,
                            GLogLevelFlags log_level,
                            const gchar   *message,
 @@ -986,38 +1016,237 @@
         return 0;
  }
  
 -/* Renames pathname by removing the .new extension */
 -static gboolean atomic_update(const gchar *pathname, GError **error)
 +static gboolean add_atomic_update(const gchar *pathname, GError **error)
  {
 -       gboolean ret = FALSE;
 -       gchar *new_name = NULL;
         int len;
  
         len = strlen(pathname);
 +       g_return_val_if_fail(len > 4, FALSE);
 +       g_return_val_if_fail(strcmp(pathname + len - 4, ".new") == 0, FALSE);
 +
 +       gchar *dupname = g_strdup(pathname);
 +       g_hash_table_add(rename_set, dupname);
  
 +       return TRUE;
 +}
 +
 +static gboolean atomic_update_file(const gchar *pathname, GError **error){
 +       gboolean ret = FALSE;
 +       int len;
 +       gchar *new_name = NULL;
 +
 +       len = strlen(pathname);
 +       g_return_val_if_fail(len > 4, FALSE);
         g_return_val_if_fail(strcmp(pathname + len - 4, ".new") == 0, FALSE);
  
         new_name = g_strndup(pathname, len - 4);
  
 -       if (sync_file(pathname, error) == -1)
 -               goto out;
 -
  #ifdef _WIN32
         /* we need to remove the old file first! */
         remove(new_name);
  #endif
 +
         if (rename(pathname, new_name) == -1)
         {
                 int errsv = errno;
 -               g_set_error(error, G_FILE_ERROR, g_file_error_from_errno(errsv),
 -                           "Failed to rename %s as %s: %s", pathname, new_name,
 -                           g_strerror(errsv));
 +               g_set_error(error, G_FILE_ERROR,
 +                       g_file_error_from_errno(errsv),
 +                       "Failed to rename %s as %s: %s",
 +                       pathname, new_name, g_strerror(errsv));
                 goto out;
         }
  
         ret = TRUE;
  out:
         g_free(new_name);
 +
 +       return ret;
 +}
 +
 +#ifdef __LINUX__AIO_ABI_H
 +static gboolean linux_aio_wait(
 +       aio_context_t aio_context, unsigned *aio_waiting, unsigned aio_max_cnt,
 +       struct io_event *io_event_array, GError **error
 +){
 +       int fin_cnt;
 +       int i;
 +
 +       fin_cnt = io_getevents(aio_context,
 +               1, aio_max_cnt, io_event_array, NULL);
 +       if(fin_cnt < 0){
 +               int errsv = errno;
 +               if(errsv == EINTR){
 +                       return TRUE;
 +               }
 +               g_set_error(error, G_FILE_ERROR,
 +                       g_file_error_from_errno(errsv),
 +                       "io_getevents failed: %s", g_strerror(errsv));
 +               return FALSE;
 +       }
 +
 +       // rename file if success
 +       for(i = 0; i < fin_cnt; i++){
 +               struct iocb *finished = (struct iocb *)io_event_array[i].obj;
 +               gchar *pathname = (gchar *)finished->aio_data;
 +
 +               close(finished->aio_fildes);
 +               finished->aio_data = 0;         // mark fd closed
 +
 +               if(io_event_array[i].res){      // error
 +                       int errsv = errno;
 +                       if(errsv == EIO){
 +                               continue;
 +                       }
 +                       g_set_error(error, G_FILE_ERROR,
 +                               g_file_error_from_errno(errsv),
 +                               "io_getevents failed: %s", g_strerror(errsv));
 +                       return FALSE;
 +               }
 +
 +               if(!atomic_update_file(pathname, error)){
 +                       return FALSE;
 +               }
 +
 +               // remove finished element from list
 +               g_hash_table_remove(rename_set, pathname);
 +       }
 +       *aio_waiting -= fin_cnt;
 +
 +       return TRUE;
 +}
 +static gboolean finish_atomic_update_linux_aio(GError **error){
 +       gboolean ret = TRUE;
 +       guint fsync_cnt = g_hash_table_size(rename_set);
 +       unsigned aio_max_cnt = fsync_cnt;
 +       aio_context_t aio_context = 0;
 +       struct iocb *iocb_array = NULL;
 +       struct io_event *io_event_array = NULL;
 +       guint aio_submited = 0;
 +       unsigned aio_waiting = 0;
 +       int evfd = -1;
 +       GList *listp, *listh = NULL;
 +       int i;
 +
 +       if(!sync_enabled() || fsync_cnt <= 1){
 +               goto noaio;
 +       }
 +
 +       while(io_setup(aio_max_cnt, &aio_context)){
 +               if(errno == EAGAIN){
 +                       aio_max_cnt = (aio_max_cnt + 1) / 2;
 +                       if(aio_max_cnt <= 1){
 +                               goto noaio;
 +                       }
 +               }else{
 +                       goto noaio;
 +               }
 +       }
 +
 +       if((evfd = eventfd(0, 0)) < 0){
 +               goto exit_aio;
 +       }
 +
 +       iocb_array = g_malloc0(sizeof(struct iocb) * fsync_cnt);
 +       io_event_array
 +               = g_malloc(sizeof(struct io_event) * aio_max_cnt);
 +
 +       listh = g_hash_table_get_keys(rename_set);
 +       for(listp = listh; listp; listp = listp->next){
 +               gchar *pathname = listp->data;
 +               struct iocb *iocbp;
 +               int fd;
 +
 +               while(aio_waiting >= aio_max_cnt){
 +                       if(!linux_aio_wait(aio_context, &aio_waiting,
 +                               aio_max_cnt, io_event_array, error)
 +                       ){
 +                               ret = FALSE;
 +                               goto exit_aio;
 +                       }
 +               }
 +
 +               // make aio request
 +               if((fd = open(pathname, O_RDWR)) < 0){
 +                       continue;
 +               }
 +               iocbp = &iocb_array[aio_submited];
 +               iocbp->aio_data = (__u64)pathname;
 +               //iocbp->aio_rw_flags = 0;
 +               iocbp->aio_lio_opcode = IOCB_CMD_FDSYNC;
 +               //iocbp->aio_reqprio = 0;
 +               iocbp->aio_fildes = fd;
 +               //iocbp->aio_buf = 0;
 +               //iocbp->aio_nbytes = 0;
 +               //iocbp->aio_offset = 0;
 +               //iocbp->aio_flags = 0;
 +               iocbp->aio_resfd = evfd;
 +
 +               if(io_submit(aio_context, (long)1, &iocbp) != 1){
 +                       close(fd);
 +                       continue;
 +               }
 +
 +               aio_waiting++;
 +               aio_submited++;
 +       }
 +
 +       while(aio_waiting > 0){
 +               if(!linux_aio_wait(aio_context, &aio_waiting,
 +                       aio_max_cnt, io_event_array, error)
 +               ){
 +                       ret = FALSE;
 +                       goto exit_aio;
 +               }
 +       }
 +
 +exit_aio:
 +       io_destroy(aio_context);
 +       if(evfd >= 0){
 +               close(evfd);
 +       }
 +       for(i = 0; i < aio_submited; i++){
 +               struct iocb *iocbp = &iocb_array[i];
 +               if(iocbp->aio_data != 0){
 +                       close(iocbp->aio_fildes);
 +               }
 +       }
 +
 +noaio:
 +       g_list_free(listh);
 +       g_free(iocb_array);
 +       g_free(io_event_array);
 +
 +       return ret;
 +}
 +#endif //__LINUX__AIO_ABI_H
 +
 +static gboolean finish_atomic_update(GError **error){
 +       gboolean ret = FALSE;
 +       GHashTableIter it;
 +    gpointer k, v;
 +
 +#ifdef __LINUX__AIO_ABI_H
 +       if(!finish_atomic_update_linux_aio(error)){
 +               goto out;
 +       }
 +#endif
 +
 +       g_hash_table_iter_init(&it, rename_set);
 +       while(g_hash_table_iter_next(&it, &k, &v)){
 +               gchar *pathname = (gchar *)k;
 +
 +               if(sync_file(pathname, error)){
 +                       goto out;
 +               }
 +
 +               if(!atomic_update_file(pathname, error)){
 +                       goto out;
 +               }
 +       }
 +
 +       ret = TRUE;
 +out:
 +       g_hash_table_remove_all(rename_set);
         return ret;
  }
  
 @@ -1048,7 +1277,7 @@
         if (!save_xml_file(type->output, filename, &local_error))
                 fatal_gerror(local_error);
  
 -       if (!atomic_update(filename, &local_error))
 +       if (!add_atomic_update(filename, &local_error))
                 fatal_gerror(local_error);
  
         g_free(filename);
 @@ -3731,6 +3960,8 @@
                                           g_free, NULL);
         generic_icon_hash = g_hash_table_new_full(g_str_hash, g_str_equal,
                                                   g_free, NULL);
 +       rename_set = g_hash_table_new_full(g_str_hash, g_str_equal,
 +                                          g_free, NULL);
  
         scan_source_dir(package_dir);
         g_free(package_dir);
 @@ -3756,7 +3987,7 @@
                 write_out_glob(glob_list, globs);
                 if (!fclose_gerror(globs, error))
                         goto out;
 -               if (!atomic_update(globs_path, error))
 +               if (!add_atomic_update(globs_path, error))
                         goto out;
                 g_free(globs_path);
  
 @@ -3770,7 +4001,7 @@
                 write_out_glob2 (glob_list, globs);
                 if (!fclose_gerror(globs, error))
                         goto out;
 -               if (!atomic_update(globs_path, error))
 +               if (!add_atomic_update(globs_path, error))
                         goto out;
                 g_free(globs_path);
  
 @@ -3797,7 +4028,7 @@
                 }
                 if (!fclose_gerror(stream, error))
                         goto out;
 -               if (!atomic_update(magic_path, error))
 +               if (!add_atomic_update(magic_path, error))
                         goto out;
                 g_free(magic_path);
         }
 @@ -3813,7 +4044,7 @@
                 write_namespaces(stream);
                 if (!fclose_gerror(stream, error))
                         goto out;
 -               if (!atomic_update(ns_path, error))
 +               if (!add_atomic_update(ns_path, error))
                         goto out;
                 g_free(ns_path);
         }
 @@ -3829,7 +4060,7 @@
                 write_subclasses(stream);
                 if (!fclose_gerror(stream, error))
                         goto out;
 -               if (!atomic_update(path, error))
 +               if (!add_atomic_update(path, error))
                         goto out;
                 g_free(path);
         }
 @@ -3845,7 +4076,7 @@
                 write_aliases(stream);
                 if (!fclose_gerror(stream, error))
                         goto out;
 -               if (!atomic_update(path, error))
 +               if (!add_atomic_update(path, error))
                         goto out;
                 g_free(path);
         }
 @@ -3861,7 +4092,7 @@
                 write_types(stream);
                 if (!fclose_gerror(stream, error))
                         goto out;
 -               if (!atomic_update(path, error))
 +               if (!add_atomic_update(path, error))
                         goto out;
                 g_free(path);
         }
 @@ -3877,7 +4108,7 @@
                 write_icons(generic_icon_hash, stream);
                 if (!fclose_gerror(stream, error))
                         goto out;
 -               if (!atomic_update(icon_path, error))
 +               if (!add_atomic_update(icon_path, error))
                         goto out;
                 g_free(icon_path);
         }
 @@ -3893,7 +4124,7 @@
                 write_icons(icon_hash, stream);
                 if (!fclose_gerror(stream, error))
                         goto out;
 -               if (!atomic_update(icon_path, error))
 +               if (!add_atomic_update(icon_path, error))
                         goto out;
                 g_free(icon_path);
         }
 @@ -3918,7 +4149,7 @@
                 }
                 if (!fclose_gerror(stream, error))
                         goto out;
 -               if (!atomic_update(path, error))
 +               if (!add_atomic_update(path, error))
                         goto out;
                 g_free(path);
         }
 @@ -3934,7 +4165,7 @@
                 write_cache(stream);
                 if (!fclose_gerror(stream, error))
                         goto out;
 -               if (!atomic_update(path, error))
 +               if (!add_atomic_update(path, error))
                         goto out;
                 g_free(path);
         }
 @@ -3951,11 +4182,15 @@
                           VERSION "\n");
                 if (!fclose_gerror(stream, error))
                         goto out;
 -               if (!atomic_update(path, error))
 +               if (!add_atomic_update(path, error))
                         goto out;
                 g_free(path);
         }
  
 +       if (!finish_atomic_update(error)){
 +               goto out;
 +       }
 +
         g_ptr_array_foreach(magic_array, (GFunc)magic_free, NULL);
         g_ptr_array_free(magic_array, TRUE);
         g_ptr_array_foreach(tree_magic_array, (GFunc)tree_magic_free, NULL);
 @@ -3968,6 +4203,7 @@
         g_hash_table_destroy(alias_hash);
         g_hash_table_destroy(icon_hash);
         g_hash_table_destroy(generic_icon_hash);
 +       g_hash_table_destroy(rename_set);
  
         check_in_path_xdg_data(mime_dir);