diff --git a/CHANGELOG.md b/CHANGELOG.md
index a3b0c16d..7b7ced9f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Fixed
 
 - Replaced `Command` with `Super` in the Linux and Windows config documentation
+- Prevent semantic and line selection from starting with the right or middle mouse button
 
 ## Version 0.2.5
 
diff --git a/src/event.rs b/src/event.rs
index 4f7f1a76..bada4f68 100644
--- a/src/event.rs
+++ b/src/event.rs
@@ -10,7 +10,7 @@ use std::env;
 
 use serde_json as json;
 use parking_lot::MutexGuard;
-use glutin::{self, ModifiersState, Event, ElementState};
+use glutin::{self, ModifiersState, Event, ElementState, MouseButton};
 use copypasta::{Clipboard, Load, Store, Buffer as ClipboardBuffer};
 use glutin::dpi::PhysicalSize;
 
@@ -245,6 +245,7 @@ pub struct Mouse {
     pub cell_side: Side,
     pub lines_scrolled: f32,
     pub block_url_launcher: bool,
+    pub last_button: MouseButton,
 }
 
 impl Default for Mouse {
@@ -263,6 +264,7 @@ impl Default for Mouse {
             cell_side: Side::Left,
             lines_scrolled: 0.0,
             block_url_launcher: false,
+            last_button: MouseButton::Other(0),
         }
     }
 }
diff --git a/src/input.rs b/src/input.rs
index 45956794..be8ed0d1 100644
--- a/src/input.rs
+++ b/src/input.rs
@@ -451,14 +451,14 @@ impl<'a, A: ActionContext + 'a> Processor<'a, A> {
         }
     }
 
-    pub fn on_mouse_double_click(&mut self) {
-        if let Some(point) = self.ctx.mouse_coords() {
+    pub fn on_mouse_double_click(&mut self, button: MouseButton) {
+        if let (Some(point) , MouseButton::Left) = (self.ctx.mouse_coords(), button) {
             self.ctx.semantic_selection(point);
         }
     }
 
-    pub fn on_mouse_triple_click(&mut self) {
-        if let Some(point) = self.ctx.mouse_coords() {
+    pub fn on_mouse_triple_click(&mut self, button: MouseButton) {
+        if let (Some(point), MouseButton::Left) = (self.ctx.mouse_coords(), button) {
             self.ctx.line_selection(point);
         }
     }
@@ -468,15 +468,21 @@ impl<'a, A: ActionContext + 'a> Processor<'a, A> {
         let elapsed = self.ctx.mouse().last_click_timestamp.elapsed();
         self.ctx.mouse_mut().last_click_timestamp = now;
 
+        let button_changed = self.ctx.mouse().last_button != button;
+
         self.ctx.mouse_mut().click_state = match self.ctx.mouse().click_state {
-            ClickState::Click if elapsed < self.mouse_config.double_click.threshold => {
+            ClickState::Click
+                if !button_changed && elapsed < self.mouse_config.double_click.threshold =>
+            {
                 self.ctx.mouse_mut().block_url_launcher = true;
-                self.on_mouse_double_click();
+                self.on_mouse_double_click(button);
                 ClickState::DoubleClick
             },
-            ClickState::DoubleClick if elapsed < self.mouse_config.triple_click.threshold => {
+            ClickState::DoubleClick
+                if !button_changed && elapsed < self.mouse_config.triple_click.threshold =>
+            {
                 self.ctx.mouse_mut().block_url_launcher = true;
-                self.on_mouse_triple_click();
+                self.on_mouse_triple_click(button);
                 ClickState::TripleClick
             },
             _ => {
@@ -659,6 +665,8 @@ impl<'a, A: ActionContext + 'a> Processor<'a, A> {
             },
             ElementState::Released => self.on_mouse_release(button, modifiers),
         }
+
+        self.ctx.mouse_mut().last_button = button;
     }
 
     /// Process key input
@@ -872,6 +880,7 @@ mod tests {
         {
             name: $name:ident,
             initial_state: $initial_state:expr,
+            initial_button: $initial_button:expr,
             input: $input:expr,
             end_state: $end_state:pat,
             last_action: $last_action:expr
@@ -893,6 +902,7 @@ mod tests {
 
                 let mut mouse = Mouse::default();
                 mouse.click_state = $initial_state;
+                mouse.last_button = $initial_button;
 
                 let mut selection = None;
 
@@ -961,6 +971,7 @@ mod tests {
     test_clickstate! {
         name: single_click,
         initial_state: ClickState::None,
+        initial_button: MouseButton::Other(0),
         input: Event::WindowEvent {
             event: WindowEvent::MouseInput {
                 state: ElementState::Pressed,
@@ -977,6 +988,7 @@ mod tests {
     test_clickstate! {
         name: double_click,
         initial_state: ClickState::Click,
+        initial_button: MouseButton::Left,
         input: Event::WindowEvent {
             event: WindowEvent::MouseInput {
                 state: ElementState::Pressed,
@@ -993,6 +1005,7 @@ mod tests {
     test_clickstate! {
         name: triple_click,
         initial_state: ClickState::DoubleClick,
+        initial_button: MouseButton::Left,
         input: Event::WindowEvent {
             event: WindowEvent::MouseInput {
                 state: ElementState::Pressed,
@@ -1006,6 +1019,23 @@ mod tests {
         last_action: MultiClick::TripleClick
     }
 
+    test_clickstate! {
+        name: multi_click_separate_buttons,
+        initial_state: ClickState::DoubleClick,
+        initial_button: MouseButton::Left,
+        input: Event::WindowEvent {
+            event: WindowEvent::MouseInput {
+                state: ElementState::Pressed,
+                button: MouseButton::Right,
+                device_id: unsafe { ::std::mem::transmute_copy(&0) },
+                modifiers: ModifiersState::default(),
+            },
+            window_id: unsafe { ::std::mem::transmute_copy(&0) },
+        },
+        end_state: ClickState::Click,
+        last_action: MultiClick::None
+    }
+
     test_process_binding! {
         name: process_binding_nomode_shiftmod_require_shift,
         binding: Binding { trigger: KEY, mods: ModifiersState { shift: true, ctrl: false, alt: false, logo: false }, action: Action::from("\x1b[1;2D"), mode: TermMode::NONE, notmode: TermMode::NONE },