Merge branch 'master' into master

Dockerfile

@@ -34,7 +34,7 @@ VOLUME /youtube
 
 # start
 WORKDIR /app
-EXPOSE 80
+EXPOSE 8000
 
 RUN chmod +x ./run.sh
 

README.md

@@ -82,7 +82,8 @@ Detect the YouTube ID from filename, this accepts the default yt-dlp naming conv
 
 
 ## Potential pitfalls
-**Elastic Search** in Docker requires the kernel setting of the host machine `vm.max_map_count` to be set to least 262144.
+### vm.max_map_count
+**Elastic Search** in Docker requires the kernel setting of the host machine `vm.max_map_count` to be set to at least 262144.
 
 To temporary set the value run:  
 ```
@@ -94,6 +95,13 @@ To apply the change permanently depends on your host operating system:
 - On Arch based systems create a file */etc/sysctl.d/max_map_count.conf* with the content `vm.max_map_count = 262144`. 
 - On any other platform look up in the documentation on how to pass kernel parameters.
 
+### Permissions for elasticsearch
+If you see a message similar to `AccessDeniedException[/usr/share/elasticsearch/data/nodes]` when initially starting elasticsearch, that means the container is not allowed to write files to the volume.  
+That's most likely the case when you run `docker-compose` as an unprivileged user. To fix that issue, shutdown the container and on your host machine run:
+```
+chown 1000:0 /path/to/mount/point
+```
+This will match the permissions with the **UID** and **GID** of elasticsearch within the container and should fix the issue.
 
 ## Roadmap
 This should be considered as a **minimal viable product**, there is an extensive list of future functions and improvements planned.

run.sh

@@ -14,7 +14,7 @@ until curl "$ES_URL" -fs; do
 done
 
 python manage.py migrate
-python manage.py collectstatic
+python manage.py collectstatic --noinput -c
 nginx &
 celery -A home.tasks worker --loglevel=INFO &
 uwsgi --ini uwsgi.ini

tubearchivist/home/__init__.py

@@ -22,8 +22,8 @@ def sync_redis_state():
 
 
 def make_folders():
-    """ make needed folders here to avoid letting docker messing it up """
-    folders = ['download', 'channels', 'videos', 'import']
+    """ make needed cache folders here so docker doesn't mess it up """
+    folders = ['download', 'channels', 'videos', 'import', 'backup']
     config = AppConfig().config
     cache_dir = config['application']['cache_dir']
     for folder in folders:

tubearchivist/home/src/index_management.py

@@ -8,6 +8,7 @@ Functionality:
 
 import json
 import os
+import zipfile
 
 from datetime import datetime
 
@@ -375,6 +376,7 @@ class ElasticBackup:
         self.config = AppConfig().config
         self.index_config = index_config
         self.timestamp = datetime.now().strftime('%Y%m%d')
+        self.backup_files = []
 
     def get_all_documents(self, index_name):
         """ export all documents of a single index """
@@ -389,7 +391,7 @@ class ElasticBackup:
         data = {
             "query": {"match_all": {}},
             "size": 100, "pit": {"id": pit_id, "keep_alive": "1m"},
-            "sort": [ {"_id": {"order": "asc"}} ]
+            "sort": [{"_id": {"order": "asc"}}]
         }
         query_str = json.dumps(data)
         url = es_url + '/_search'
@@ -422,7 +424,7 @@ class ElasticBackup:
         for document in all_results:
             document_id = document['_id']
             es_index = document['_index']
-            action = { "index" : { "_index": es_index, "_id": document_id } }
+            action = {"index": {"_index": es_index, "_id": document_id}}
             source = document['_source']
             bulk_list.append(json.dumps(action))
             bulk_list.append(json.dumps(source))
@@ -433,14 +435,44 @@ class ElasticBackup:
 
         return file_content
 
-    def write_json_file(self, file_content, index_name):
-        """ write json file to disk """
+    def write_es_json(self, file_content, index_name):
+        """ write nd json file for es _bulk API to disk """
         cache_dir = self.config['application']['cache_dir']
-        file_name = f'ta_{index_name}-{self.timestamp}.json'
-        file_path = os.path.join(cache_dir, file_name)
+        file_name = f'es_{index_name}-{self.timestamp}.json'
+        file_path = os.path.join(cache_dir, 'backup', file_name)
         with open(file_path, 'w', encoding='utf-8') as f:
             f.write(file_content)
 
+        self.backup_files.append(file_path)
+
+    def write_ta_json(self, all_results, index_name):
+        """ write generic json file to disk """
+        cache_dir = self.config['application']['cache_dir']
+        file_name = f'ta_{index_name}-{self.timestamp}.json'
+        file_path = os.path.join(cache_dir, 'backup', file_name)
+        to_write = [i['_source'] for i in all_results]
+        file_content = json.dumps(to_write)
+        with open(file_path, 'w', encoding='utf-8') as f:
+            f.write(file_content)
+
+        self.backup_files.append(file_path)
+
+    def zip_it(self):
+        """ pack it up into single zip file """
+        cache_dir = self.config['application']['cache_dir']
+        file_name = f'ta_backup-{self.timestamp}.zip'
+        backup_file = os.path.join(cache_dir, 'backup', file_name)
+
+        with zipfile.ZipFile(
+                    backup_file, 'w', compression=zipfile.ZIP_DEFLATED
+                ) as zip_f:
+            for backup_file in self.backup_files:
+                zip_f.write(backup_file)
+
+        # cleanup
+        for backup_file in self.backup_files:
+            os.remove(backup_file)
+
     def post_bulk_restore(self, file_name):
         """ send bulk to es """
         cache_dir = self.config['application']['cache_dir']
@@ -475,7 +507,10 @@ def backup_all_indexes():
         index_name = index['index_name']
         all_results = backup_handler.get_all_documents(index_name)
         file_content = backup_handler.build_bulk(all_results)
-        backup_handler.write_json_file(file_content, index_name)
+        backup_handler.write_es_json(file_content, index_name)
+        backup_handler.write_ta_json(all_results, index_name)
+
+    backup_handler.zip_it()
 
 
 def restore_from_backup():

tubearchivist/home/tasks.py

@@ -15,6 +15,7 @@ from home.src.download import (
 )
 from home.src.config import AppConfig
 from home.src.reindex import reindex_old_documents, ManualImport
+from home.src.index_management import backup_all_indexes
 from home.src.helper import get_lock
 
 
@@ -54,8 +55,7 @@ def download_pending():
 @shared_task
 def download_single(youtube_id):
     """ start download single video now """
-    to_download = [youtube_id]
-    download_handler = VideoDownloader(to_download)
+    download_handler = VideoDownloader([youtube_id])
     download_handler.download_list()
 
 
@@ -93,3 +93,9 @@ def run_manual_import():
     finally:
         if have_lock:
             my_lock.release()
+
+@shared_task
+def run_backup():
+    """ called from settings page, dump backup to zip file """
+    backup_all_indexes()
+    print('backup finished')

tubearchivist/home/templates/home/downloads.html

@@ -29,7 +29,7 @@
 <h2>Download queue</h2>
 <div>
     {% if pending %}
-        <h3>Total pending downloads: {{ pending|length }}</h3>
+        <h3>Total pending downloads: {{ max_hits }}</h3>
         {% for video in pending %}
             <div class="dl-item" id="dl-{{ video.youtube_id }}">
                 <div class="dl-thumb">
@@ -44,7 +44,7 @@
                     {% endif %}
                     <p>Published: {{ video.published }} | Duration: {{ video.duration }} | {{ video.youtube_id }}</p>
                     <button data-id="{{ video.youtube_id }}" onclick="toIgnore(this)">Ignore</button>
-                    <button data-id="{{ video.youtube_id }}" onclick="downloadNow(this)">Download now</button>
+                    <button id="{{ video.youtube_id }}" data-id="{{ video.youtube_id }}" onclick="downloadNow(this)">Download now</button>
                 </div>
             </div>
         {% endfor %}

tubearchivist/home/templates/home/settings.html

@@ -114,17 +114,24 @@
 </div>
 <div class="settings-group">
     <h2>Manual media files import.</h2>
-    <p>Add files to the cache/import folder. Make sure to follow the instructions on <a href="https://github.com/bbilly1/tubearchivist#import-your-existing-library" target="_blank">Github</a>.</p>
+    <p>Add files to the <span class="settings-current">cache/import</span> folder. Make sure to follow the instructions on <a href="https://github.com/bbilly1/tubearchivist#import-your-existing-library" target="_blank">Github</a>.</p>
     <div id="manual-import">
         <button onclick="manualImport()">Start import</button>
     </div>
 </div>
 <div class="settings-group">
-    <p>Rescan filesystem.</p>
+    <h2>Backup database</h2>
+    <p>Export your database to a zip file stored at <span class="settings-current">cache/backup</span>.</p>
+    <div id="db-backup">
+        <button onclick="dbBackup()">Start backup</button>
+    </div>
+</div>
+<div class="settings-group">
+    <p>Restore from backup.</p>
     <i>Coming soon</i>
 </div>
 <div class="settings-group">
-    <p>Backup database.</p>
+    <p>Rescan filesystem.</p>
     <i>Coming soon</i>
 </div>
 {% endblock content %}

tubearchivist/home/views.py

@@ -31,7 +31,8 @@ from home.tasks import (
     download_pending,
     extrac_dl,
     download_single,
-    run_manual_import
+    run_manual_import,
+    run_backup
 )
 
 
@@ -147,20 +148,50 @@ class DownloadView(View):
     takes POST for downloading youtube links
     """
 
-    @staticmethod
-    def get(request):
+    def get(self, request):
         """ handle get requests """
         config = AppConfig().config
         colors = config['application']['colors']
-        pending_handler = PendingList()
-        all_pending, _ = pending_handler.get_all_pending()
+
+        page_get = int(request.GET.get('page', 0))
+        pagination_handler = Pagination(page_get)
+
+        url = config['application']['es_url'] + '/ta_download/_search'
+        data = self.build_data(pagination_handler)
+        search = SearchHandler(url, data, cache=False)
+
+        videos_hits = search.get_data()
+        max_hits = search.max_hits
+
+        if videos_hits:
+            all_pending = [i['source'] for i in videos_hits]
+            pagination_handler.validate(max_hits)
+            pagination = pagination_handler.pagination
+        else:
+            all_pending = False
+            pagination = False
+
         context = {
             'pending': all_pending,
+            'max_hits': max_hits,
+            'pagination': pagination,
             'title': 'Downloads',
             'colors': colors
         }
         return render(request, 'home/downloads.html', context)
 
+    @staticmethod
+    def build_data(pagination_handler):
+        """ build data dict for search """
+        page_size = pagination_handler.pagination['page_size']
+        page_from = pagination_handler.pagination['page_from']
+        data = {
+            "size": page_size, "from": page_from,
+            "query": {"term": {"status": {"value": "pending"}}},
+            "sort": [{"timestamp": {"order": "desc"}}]
+        }
+        return data
+
     @staticmethod
     def post(request):
         """ handle post requests """
@@ -442,7 +473,8 @@ class PostData:
     VALID_KEYS = [
         "watched", "rescan_pending", "ignore", "dl_pending",
         "unsubscribe", "sort_order", "hide_watched", "show_subed_only",
-        "channel-search", "video-search", "dlnow", "manual-import"
+        "channel-search", "video-search", "dlnow", "manual-import",
+        "db-backup"
     ]
 
     def __init__(self, post_dict):
@@ -510,10 +542,13 @@ class PostData:
             elif task == 'dlnow':
                 youtube_id = item['status']
                 print('downloading: ' + youtube_id)
-                download_single(youtube_id)
+                download_single.delay(youtube_id=youtube_id)
             elif task == 'manual-import':
                 print('starting manual import')
                 run_manual_import.delay()
+            elif task == 'db-backup':
+                print('backing up database')
+                run_backup.delay()
         return {'success': True}
 
     def search_channels(self, search_query):

tubearchivist/static/script.js

@@ -72,8 +72,8 @@ function toIgnore(button) {
 function downloadNow(button) {
     var youtube_id = button.getAttribute('data-id');
     var payload = JSON.stringify({'dlnow': youtube_id});
-    animate('download-icon', 'bounce-img');
     sendPost(payload);
+    document.getElementById(youtube_id).remove();
     setTimeout(function(){
         handleInterval();
     }, 500);
@@ -91,6 +91,17 @@ function manualImport() {
     toReplace.appendChild(message);
 }
 
+function dbBackup() {
+    var payload = JSON.stringify({'db-backup': true});
+    sendPost(payload)
+    // clear button
+    var message = document.createElement('p');
+    message.innerText = 'backing up archive';
+    var toReplace = document.getElementById('db-backup');
+    toReplace.innerHTML = '';
+    toReplace.appendChild(message);
+}
+
 // player
 function createPlayer(button) {
     var mediaUrl = button.getAttribute('data-src');