diff --git a/app/src/main/java/eu/kanade/tachiyomi/ui/library/LibraryCategoryFragment.kt b/app/src/main/java/eu/kanade/tachiyomi/ui/library/LibraryCategoryFragment.kt
index 42e67a8feb..0a6e689356 100644
--- a/app/src/main/java/eu/kanade/tachiyomi/ui/library/LibraryCategoryFragment.kt
+++ b/app/src/main/java/eu/kanade/tachiyomi/ui/library/LibraryCategoryFragment.kt
@@ -2,6 +2,8 @@ package eu.kanade.tachiyomi.ui.library
 
 import android.content.res.Configuration
 import android.os.Bundle
+import android.support.v7.widget.LinearLayoutManager
+import android.support.v7.widget.RecyclerView
 import android.view.LayoutInflater
 import android.view.View
 import android.view.ViewGroup
@@ -102,6 +104,15 @@ class LibraryCategoryFragment : BaseFragment(), FlexibleViewHolder.OnListItemCli
             }
         }
 
+        recycler.addOnScrollListener(object : RecyclerView.OnScrollListener() {
+            override fun onScrollStateChanged(recycler: RecyclerView, newState: Int) {
+                // Disable swipe refresh when view is not at the top
+                val firstPos = (recycler.layoutManager as LinearLayoutManager)
+                        .findFirstCompletelyVisibleItemPosition()
+                swipe_refresh.isEnabled = firstPos == 0
+            }
+        })
+
         swipe_refresh.setOnRefreshListener {
             if (!LibraryUpdateService.isRunning(activity)) {
                 libraryPresenter.categories.getOrNull(position)?.let {